Backport/sanikachavan5/secvuln 8628 add charset wherever applicable

.changelog/22598.txt

@@ -0,0 +1,3 @@
+```release-note:security
+api: add charset in all applicable content-types.
+```

agent/checks/check_test.go

@@ -20,11 +20,12 @@ import (
 	"testing"
 	"time"
 
-	"github.com/hashicorp/go-uuid"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/net/http2"
 	"golang.org/x/net/http2/h2c"
 
+	"github.com/hashicorp/go-uuid"
+
 	"github.com/hashicorp/consul/agent/mock"
 	"github.com/hashicorp/consul/agent/structs"
 	"github.com/hashicorp/consul/api"
@@ -689,7 +690,7 @@ func TestCheckHTTPBody(t *testing.T) {
 	}{
 		{desc: "get body", method: "GET", body: "hello world"},
 		{desc: "post body", method: "POST", body: "hello world"},
-		{desc: "post json body", header: http.Header{"Content-Type": []string{"application/json"}}, method: "POST", body: "{\"foo\":\"bar\"}"},
+		{desc: "post json body", header: http.Header{"Content-Type": []string{"application/json; charset=utf-8"}}, method: "POST", body: "{\"foo\":\"bar\"}"},
 	}
 
 	for _, tt := range tests {
@@ -1561,7 +1562,7 @@ func TestCheck_Docker(t *testing.T) {
 			handlers: map[string]http.HandlerFunc{
 				"POST /containers/123/exec": func(w http.ResponseWriter, r *http.Request) {
 					w.WriteHeader(201)
-					w.Header().Set("Content-Type", "application/json")
+					w.Header().Set("Content-Type", "application/json; charset=utf-8")
 					fmt.Fprint(w, `this is not json`)
 				},
 			},
@@ -1573,7 +1574,7 @@ func TestCheck_Docker(t *testing.T) {
 			handlers: map[string]http.HandlerFunc{
 				"POST /containers/123/exec": func(w http.ResponseWriter, r *http.Request) {
 					w.WriteHeader(201)
-					w.Header().Set("Content-Type", "application/json")
+					w.Header().Set("Content-Type", "application/json; charset=utf-8")
 					fmt.Fprint(w, `{"Id":"456"}`)
 				},
 				"POST /exec/456/start": func(w http.ResponseWriter, r *http.Request) {
@@ -1588,7 +1589,7 @@ func TestCheck_Docker(t *testing.T) {
 			handlers: map[string]http.HandlerFunc{
 				"POST /containers/123/exec": func(w http.ResponseWriter, r *http.Request) {
 					w.WriteHeader(201)
-					w.Header().Set("Content-Type", "application/json")
+					w.Header().Set("Content-Type", "application/json; charset=utf-8")
 					fmt.Fprint(w, `{"Id":"456"}`)
 				},
 				"POST /exec/456/start": func(w http.ResponseWriter, r *http.Request) {
@@ -1603,7 +1604,7 @@ func TestCheck_Docker(t *testing.T) {
 			handlers: map[string]http.HandlerFunc{
 				"POST /containers/123/exec": func(w http.ResponseWriter, r *http.Request) {
 					w.WriteHeader(201)
-					w.Header().Set("Content-Type", "application/json")
+					w.Header().Set("Content-Type", "application/json; charset=utf-8")
 					fmt.Fprint(w, `{"Id":"456"}`)
 				},
 				"POST /exec/456/start": func(w http.ResponseWriter, r *http.Request) {
@@ -1619,7 +1620,7 @@ func TestCheck_Docker(t *testing.T) {
 			handlers: map[string]http.HandlerFunc{
 				"POST /containers/123/exec": func(w http.ResponseWriter, r *http.Request) {
 					w.WriteHeader(201)
-					w.Header().Set("Content-Type", "application/json")
+					w.Header().Set("Content-Type", "application/json; charset=utf-8")
 					fmt.Fprint(w, `{"Id":"456"}`)
 				},
 				"POST /exec/456/start": func(w http.ResponseWriter, r *http.Request) {
@@ -1638,7 +1639,7 @@ func TestCheck_Docker(t *testing.T) {
 			handlers: map[string]http.HandlerFunc{
 				"POST /containers/123/exec": func(w http.ResponseWriter, r *http.Request) {
 					w.WriteHeader(201)
-					w.Header().Set("Content-Type", "application/json")
+					w.Header().Set("Content-Type", "application/json; charset=utf-8")
 					fmt.Fprint(w, `{"Id":"456"}`)
 				},
 				"POST /exec/456/start": func(w http.ResponseWriter, r *http.Request) {
@@ -1658,7 +1659,7 @@ func TestCheck_Docker(t *testing.T) {
 			handlers: map[string]http.HandlerFunc{
 				"POST /containers/123/exec": func(w http.ResponseWriter, r *http.Request) {
 					w.WriteHeader(201)
-					w.Header().Set("Content-Type", "application/json")
+					w.Header().Set("Content-Type", "application/json; charset=utf-8")
 					fmt.Fprint(w, `{"Id":"456"}`)
 				},
 				"POST /exec/456/start": func(w http.ResponseWriter, r *http.Request) {
@@ -1667,7 +1668,7 @@ func TestCheck_Docker(t *testing.T) {
 				},
 				"GET /exec/456/json": func(w http.ResponseWriter, r *http.Request) {
 					w.WriteHeader(200)
-					w.Header().Set("Content-Type", "application/json")
+					w.Header().Set("Content-Type", "application/json; charset=utf-8")
 					fmt.Fprint(w, `this is not json`)
 				},
 			},
@@ -1679,7 +1680,7 @@ func TestCheck_Docker(t *testing.T) {
 			handlers: map[string]http.HandlerFunc{
 				"POST /containers/123/exec": func(w http.ResponseWriter, r *http.Request) {
 					w.WriteHeader(201)
-					w.Header().Set("Content-Type", "application/json")
+					w.Header().Set("Content-Type", "application/json; charset=utf-8")
 					fmt.Fprint(w, `{"Id":"456"}`)
 				},
 				"POST /exec/456/start": func(w http.ResponseWriter, r *http.Request) {
@@ -1688,7 +1689,7 @@ func TestCheck_Docker(t *testing.T) {
 				},
 				"GET /exec/456/json": func(w http.ResponseWriter, r *http.Request) {
 					w.WriteHeader(200)
-					w.Header().Set("Content-Type", "application/json")
+					w.Header().Set("Content-Type", "application/json; charset=utf-8")
 					fmt.Fprint(w, `{"ExitCode":0}`)
 				},
 			},
@@ -1700,7 +1701,7 @@ func TestCheck_Docker(t *testing.T) {
 			handlers: map[string]http.HandlerFunc{
 				"POST /containers/123/exec": func(w http.ResponseWriter, r *http.Request) {
 					w.WriteHeader(201)
-					w.Header().Set("Content-Type", "application/json")
+					w.Header().Set("Content-Type", "application/json; charset=utf-8")
 					fmt.Fprint(w, `{"Id":"456"}`)
 				},
 				"POST /exec/456/start": func(w http.ResponseWriter, r *http.Request) {
@@ -1709,7 +1710,7 @@ func TestCheck_Docker(t *testing.T) {
 				},
 				"GET /exec/456/json": func(w http.ResponseWriter, r *http.Request) {
 					w.WriteHeader(200)
-					w.Header().Set("Content-Type", "application/json")
+					w.Header().Set("Content-Type", "application/json; charset=utf-8")
 					fmt.Fprint(w, `{"ExitCode":0}`)
 				},
 			},
@@ -1721,7 +1722,7 @@ func TestCheck_Docker(t *testing.T) {
 			handlers: map[string]http.HandlerFunc{
 				"POST /containers/123/exec": func(w http.ResponseWriter, r *http.Request) {
 					w.WriteHeader(201)
-					w.Header().Set("Content-Type", "application/json")
+					w.Header().Set("Content-Type", "application/json; charset=utf-8")
 					fmt.Fprint(w, `{"Id":"456"}`)
 				},
 				"POST /exec/456/start": func(w http.ResponseWriter, r *http.Request) {
@@ -1730,7 +1731,7 @@ func TestCheck_Docker(t *testing.T) {
 				},
 				"GET /exec/456/json": func(w http.ResponseWriter, r *http.Request) {
 					w.WriteHeader(200)
-					w.Header().Set("Content-Type", "application/json")
+					w.Header().Set("Content-Type", "application/json; charset=utf-8")
 					fmt.Fprint(w, `{"ExitCode":1}`)
 				},
 			},
@@ -1742,7 +1743,7 @@ func TestCheck_Docker(t *testing.T) {
 			handlers: map[string]http.HandlerFunc{
 				"POST /containers/123/exec": func(w http.ResponseWriter, r *http.Request) {
 					w.WriteHeader(201)
-					w.Header().Set("Content-Type", "application/json")
+					w.Header().Set("Content-Type", "application/json; charset=utf-8")
 					fmt.Fprint(w, `{"Id":"456"}`)
 				},
 				"POST /exec/456/start": func(w http.ResponseWriter, r *http.Request) {
@@ -1751,7 +1752,7 @@ func TestCheck_Docker(t *testing.T) {
 				},
 				"GET /exec/456/json": func(w http.ResponseWriter, r *http.Request) {
 					w.WriteHeader(200)
-					w.Header().Set("Content-Type", "application/json")
+					w.Header().Set("Content-Type", "application/json; charset=utf-8")
 					fmt.Fprint(w, `{"ExitCode":2}`)
 				},
 			},

agent/consul/authmethod/kubeauth/testing.go

@@ -107,7 +107,7 @@ func (s *TestAPIServer) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 
-	w.Header().Set("content-type", "application/json")
+	w.Header().Set("content-type", "application/json; charset=utf-8")
 
 	if req.URL.Path == "/apis/authentication.k8s.io/v1/tokenreviews" {
 		s.handleTokenReview(w, req)

agent/hcp/testing.go

@@ -119,7 +119,7 @@ func (s *MockHCPServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	}
 
 	log.Printf("OK 200: %s %s\n", r.Method, r.URL.Path)
-	w.Header().Set("content-type", "application/json")
+	w.Header().Set("content-type", "application/json; charset=utf-8")
 	w.WriteHeader(http.StatusOK)
 	w.Write(bs)
 }
@@ -137,7 +137,7 @@ func enforceMethod(w http.ResponseWriter, r *http.Request, methods []string) boo
 }
 
 func mockTokenResponse(w http.ResponseWriter) {
-	w.Header().Set("Content-Type", "application/json")
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
 	w.WriteHeader(http.StatusOK)
 
 	w.Write([]byte(`{"access_token": "token", "token_type": "Bearer"}`))

agent/http_test.go

@@ -24,12 +24,13 @@ import (
 	"time"
 
 	"github.com/NYTimes/gziphandler"
-	"github.com/hashicorp/go-cleanhttp"
-	"github.com/hashicorp/go-hclog"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/net/http2"
 
+	"github.com/hashicorp/go-cleanhttp"
+	"github.com/hashicorp/go-hclog"
+
 	"github.com/hashicorp/consul/agent/config"
 	"github.com/hashicorp/consul/agent/consul"
 	"github.com/hashicorp/consul/agent/structs"
@@ -638,7 +639,7 @@ func TestHTTPAPIResponseHeaders(t *testing.T) {
 	`)
 	defer a.Shutdown()
 
-	requireHasHeadersSet(t, a, "/v1/agent/self", "application/json")
+	requireHasHeadersSet(t, a, "/v1/agent/self", "application/json; charset=utf-8")
 
 	// Check the Index page that just renders a simple message with UI disabled
 	// also gets the right headers.
@@ -686,7 +687,7 @@ func TestHTTPAPIValidateContentTypeHeaders(t *testing.T) {
 			method:              http.MethodPost,
 			endpoint:            "/v1/peering/token",
 			requestBody:         bytes.NewBuffer([]byte("test")),
-			expectedContentType: "application/json",
+			expectedContentType: "application/json; charset=utf-8",
 		},
 	}
 
@@ -784,7 +785,7 @@ func TestErrorContentTypeHeaderSet(t *testing.T) {
 	`)
 	defer a.Shutdown()
 
-	requireHasHeadersSet(t, a, "/fake-path-doesn't-exist", "application/json")
+	requireHasHeadersSet(t, a, "/fake-path-doesn't-exist", "application/json; charset=utf-8")
 }
 
 func TestAcceptEncodingGzip(t *testing.T) {

agent/kvs_endpoint.go

@@ -94,7 +94,7 @@ func (s *HTTPHandlers) KVSGet(resp http.ResponseWriter, req *http.Request, args
 	if _, ok := params["raw"]; ok && method == "KVS.Get" {
 		body := out.Entries[0].Value
 		resp.Header().Set("Content-Length", strconv.FormatInt(int64(len(body)), 10))
-		resp.Header().Set("Content-Type", "text/plain")
+		resp.Header().Set("Content-Type", "text/plain; charset=utf-8")
 		resp.Header().Set("X-Content-Type-Options", "nosniff")
 		resp.Header().Set("Content-Security-Policy", "sandbox")
 		resp.Write(body)

agent/kvs_endpoint_test.go

@@ -11,9 +11,8 @@ import (
 	"reflect"
 	"testing"
 
-	"github.com/hashicorp/consul/testrpc"
-
 	"github.com/hashicorp/consul/agent/structs"
+	"github.com/hashicorp/consul/testrpc"
 )
 
 func TestKVSEndpoint_PUT_GET_DELETE(t *testing.T) {
@@ -458,7 +457,7 @@ func TestKVSEndpoint_GET_Raw(t *testing.T) {
 	if len(contentTypeHdr) != 1 {
 		t.Fatalf("expected 1 value for Content-Type header, got %d: %+v", len(contentTypeHdr), contentTypeHdr)
 	}
-	if contentTypeHdr[0] != "text/plain" {
+	if contentTypeHdr[0] != "text/plain; charset=utf-8" {
 		t.Fatalf("expected Content-Type header to be \"text/plain\", got %q", contentTypeHdr[0])
 	}
 

agent/txn_endpoint.go

@@ -403,7 +403,7 @@ func (s *HTTPHandlers) Txn(resp http.ResponseWriter, req *http.Request) (interfa
 			return nil, err
 		}
 
-		resp.Header().Set("Content-Type", "application/json")
+		resp.Header().Set("Content-Type", "application/json; charset=utf-8")
 		resp.WriteHeader(http.StatusConflict)
 		resp.Write(buf)
 		return nil, nil

api/api.go

@@ -1033,12 +1033,11 @@ func (r *request) toHTTP() (*http.Request, error) {
 	req.Header = r.header
 
 	// Content-Type must always be set when a body is present
-	// See https://github.com/hashicorp/consul/issues/10011
 	if req.Body != nil && req.Header.Get("Content-Type") == "" {
-		req.Header.Set("Content-Type", "application/json")
+		req.Header.Set("Content-Type", "application/json; charset=utf-8")
 	}
 
-	// Setup auth
+	// Check for a token
 	if r.config.HttpAuth != nil {
 		req.SetBasicAuth(r.config.HttpAuth.Username, r.config.HttpAuth.Password)
 	}

api/api_test.go

@@ -935,11 +935,11 @@ func TestAPI_Headers(t *testing.T) {
 
 	_, _, err = kv.Get("test-headers", nil)
 	require.NoError(t, err)
-	require.Equal(t, "application/json", request.Header.Get("Content-Type"))
+	require.Equal(t, "application/json; charset=utf-8", request.Header.Get("Content-Type"))
 
 	_, err = kv.Delete("test-headers", nil)
 	require.NoError(t, err)
-	require.Equal(t, "application/json", request.Header.Get("Content-Type"))
+	require.Equal(t, "application/json; charset=utf-8", request.Header.Get("Content-Type"))
 
 	err = c.Snapshot().Restore(nil, strings.NewReader("foo"))
 	require.Error(t, err)

api/content_type.go

@@ -12,7 +12,7 @@ const (
 	contentTypeHeader = "Content-Type"
 	plainContentType  = "text/plain; charset=utf-8"
 	octetStream       = "application/octet-stream"
-	jsonContentType   = "application/json" // Default content type
+	jsonContentType   = "application/json; charset=utf-8" // Default content type
 )
 
 // ContentTypeRule defines a rule for determining the content type of an HTTP request.

command/resource/client/client.go

@@ -883,12 +883,11 @@ func (r *request) toHTTP() (*http.Request, error) {
 	req.Header = r.header
 
 	// Content-Type must always be set when a body is present
-	// See https://github.com/hashicorp/consul/issues/10011
 	if req.Body != nil && req.Header.Get("Content-Type") == "" {
-		req.Header.Set("Content-Type", "application/json")
+		req.Header.Set("Content-Type", "application/json; charset=utf-8")
 	}
 
-	// Setup auth
+	// Check for a token
 	if r.config.HttpAuth != nil {
 		req.SetBasicAuth(r.config.HttpAuth.Username, r.config.HttpAuth.Password)
 	}

internal/go-sso/oidcauth/oidcauthtest/testing.go

@@ -184,7 +184,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 
-	w.Header().Set("Content-Type", "application/json")
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
 
 	switch req.URL.Path {
 	case "/.well-known/openid-configuration":

website/content/docs/discover/vm.mdx

@@ -7,7 +7,7 @@ description: >-
 
 # Discover services on virtual machines (VMs)
 
-This page provides an overview of Consul service discovery operations on virtual machines. After you register services with Consul, you can address them using Consul DNS to perform application load balancing and static service lookups. You can also create prepared queries for dynamic service lookups and service failover.
+This page provides an overview of Consul service discovery operations on virtual machines. After you register services with Consul, you can use Consul DNS to perform application load balancing and static service lookups. You can also create prepared queries for dynamic service lookups and service failover.
 
 ## Introduction