backport of commit 7c7478a

.changelog/21655.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+xds: configures Envoy to load balance over all instances of an external service configured with hostnames when "envoy_dns_discovery_type" is set to "STRICT_DNS"
+```

.changelog/21806.txt

@@ -0,0 +1,3 @@
+```release-note:feature
+grafana: added the dashboards service-to-service dashboard, service dashboard, and consul dataplane dashboard
+```

.changelog/21816.txt

@@ -0,0 +1,9 @@
+```release-note:security
+mesh: Add `http.incoming.requestNormalization` to Mesh configuration entry to support inbound service traffic request normalization. This resolves [CVE-2024-10005](https://nvd.nist.gov/vuln/detail/CVE-2024-10005) and [CVE-2024-10006](https://nvd.nist.gov/vuln/detail/CVE-2024-10006).
+```
+```release-note:security
+mesh: Add `contains` and `ignoreCase` to L7 Intentions HTTP header matching criteria to support configuration resilient to variable casing and multiple values. This resolves [CVE-2024-10006](https://nvd.nist.gov/vuln/detail/CVE-2024-10006).
+```
+```release-note:breaking-change
+mesh: Enable Envoy `HttpConnectionManager.normalize_path` by default on inbound traffic to mesh proxies. This resolves [CVE-2024-10005](https://nvd.nist.gov/vuln/detail/CVE-2024-10005).
+```

.github/workflows/nightly-test-1.19.x.yaml

@@ -14,8 +14,15 @@ env:
   GOPRIVATE: github.com/hashicorp # Required for enterprise deps
 
 jobs:
+  check-ent:
+    runs-on: ubuntu-latest
+    if: ${{ endsWith(github.repository, '-enterprise') }}
+    steps:
+      - run: echo "Building Enterprise"
+
   frontend-test-workspace-node:
     runs-on: ubuntu-latest
+    needs: [ check-ent ]
     steps:
       - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
         with:
@@ -45,6 +52,7 @@ jobs:
 
   frontend-build-ce:
     runs-on: ubuntu-latest
+    needs: [ check-ent ]
     env:
       JOBS: 2
       CONSUL_NSPACES_ENABLED: 0
@@ -117,6 +125,7 @@ jobs:
 
   frontend-build-ent:
     runs-on: ubuntu-latest
+    needs: [ check-ent ]
     env:
       JOBS: 2
       CONSUL_NSPACES_ENABLED: 1

.github/workflows/nightly-test-1.17.x.yaml → .github/workflows/nightly-test-1.20.x.yaml

@@ -1,16 +1,16 @@
 # Copyright (c) HashiCorp, Inc.
 # SPDX-License-Identifier: MPL-2.0
 
-name: Nightly Frontend Test 1.17.x
+name: Nightly Frontend Test 1.20.x
 on:
   schedule:
     - cron: '0 4 * * *'
   workflow_dispatch: {}
 
 env:
   EMBER_PARTITION_TOTAL: 4        # Has to be changed in tandem with the matrix.partition
-  BRANCH: "release/1.17.x"
-  BRANCH_NAME: "release-1.17.x"   # Used for naming artifacts
+  BRANCH: "release/1.20.x"
+  BRANCH_NAME: "release-1.20.x"   # Used for naming artifacts
   GOPRIVATE: github.com/hashicorp # Required for enterprise deps
 
 jobs:
@@ -22,7 +22,7 @@ jobs:
 
   frontend-test-workspace-node:
     runs-on: ubuntu-latest
-    needs: [check-ent]
+    needs: [ check-ent ]
     steps:
       - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
         with:
@@ -52,7 +52,7 @@ jobs:
 
   frontend-build-ce:
     runs-on: ubuntu-latest
-    needs: [check-ent]
+    needs: [ check-ent ]
     env:
       JOBS: 2
       CONSUL_NSPACES_ENABLED: 0
@@ -125,7 +125,7 @@ jobs:
 
   frontend-build-ent:
     runs-on: ubuntu-latest
-    needs: [check-ent]
+    needs: [ check-ent ]
     env:
       JOBS: 2
       CONSUL_NSPACES_ENABLED: 1

.github/workflows/nightly-test-integrations-1.20.x.yml

@@ -241,7 +241,7 @@ jobs:
       # matrix.consul-version (i.e. whenever the highest common Envoy version across active
       # Consul versions changes). The minor Envoy version does not necessarily need to be
       # kept current for the purpose of these tests, but the major (1.N) version should be.
-      ENVOY_VERSION: 1.27.6
+      ENVOY_VERSION: 1.28.7
     steps:
       - name: Checkout code
         uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4

.github/workflows/test-integrations.yml

@@ -83,7 +83,7 @@ jobs:
       contents: read
     strategy:
       matrix:
-        nomad-version: ['v1.8.3', 'v1.7.7', 'v1.6.10']
+        nomad-version: ['v1.7.7', 'v1.6.10', 'v1.5.17']
     steps:
       - name: Checkout Nomad
         uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
@@ -166,7 +166,7 @@ jobs:
       contents: read
     strategy:
       matrix:
-        vault-version: ["1.17.5", "1.16.3", "1.15.6"]
+        vault-version: ["1.16.2", "1.15.6", "1.14.10"]
     env:
       VAULT_BINARY_VERSION: ${{ matrix.vault-version }}
     steps:
@@ -412,7 +412,7 @@ jobs:
     env:
       ENVOY_VERSION: ${{ needs.get-envoy-versions.outputs.max-envoy-version }}
       #TODO don't harcode this image name
-      CONSUL_DATAPLANE_IMAGE: "docker.io/hashicorppreview/consul-dataplane:1.6-dev-ubi"
+      CONSUL_DATAPLANE_IMAGE: "docker.mirror.hashicorp.services/hashicorppreview/consul-dataplane:1.6-dev-ubi"
     steps:
       - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
       # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos.

.release/security-scan.hcl

@@ -14,7 +14,7 @@
 
 container {
 	dependencies = true
-	alpine_secdb = true
+	osv          = true
 
 	secrets {
 		matchers {
@@ -36,9 +36,9 @@ container {
 	# periodically cleaned up to remove items that are no longer found by the scanner.
 	triage {
 		suppress {
-			# N.b. `vulnerabilites` is the correct spelling for this tool.
-			vulnerabilites = [
+			vulnerabilities = [
 				"CVE-2024-8096", # [email protected],
+				"CVE-2024-9143", # [email protected],
 			]
 			paths = [
 				"internal/tools/proto-gen-rpc-glue/e2e/consul/*",
@@ -78,8 +78,7 @@ binary {
 	# periodically cleaned up to remove items that are no longer found by the scanner.
 	triage {
 		suppress {
-			# N.b. `vulnerabilites` is the correct spelling for this tool.
-			vulnerabilites = [
+			vulnerabilities = [
 			]
 			paths = [
 				"internal/tools/proto-gen-rpc-glue/e2e/consul/*",

.release/versions.hcl

@@ -9,13 +9,10 @@ active_versions {
   version "1.20" {
     ce_active = true
   }
-  version "1.19" {
-    ce_active = true
-  }
+  version "1.19" {}
   version "1.18" {
     lts       = true
   }
-  version "1.17" {}
   version "1.15" {
     lts = true
   }

CHANGELOG.md

@@ -1,15 +1,62 @@
+## 1.20.1 (October 29, 2024)
+BREAKING CHANGES:
+
+* mesh: Enable Envoy `HttpConnectionManager.normalize_path` by default on inbound traffic to mesh proxies. This resolves [CVE-2024-10005](https://nvd.nist.gov/vuln/detail/CVE-2024-10005). [[GH-21816](https://github.com/hashicorp/consul/issues/21816)]
+
+SECURITY:
+
+* mesh: Add `contains` and `ignoreCase` to L7 Intentions HTTP header matching criteria to support configuration resilient to variable casing and multiple values. This resolves [CVE-2024-10006](https://nvd.nist.gov/vuln/detail/CVE-2024-10006). [[GH-21816](https://github.com/hashicorp/consul/issues/21816)]
+* mesh: Add `http.incoming.requestNormalization` to Mesh configuration entry to support inbound service traffic request normalization. This resolves [CVE-2024-10005](https://nvd.nist.gov/vuln/detail/CVE-2024-10005) and [CVE-2024-10006](https://nvd.nist.gov/vuln/detail/CVE-2024-10006). [[GH-21816](https://github.com/hashicorp/consul/issues/21816)]
+
+IMPROVEMENTS:
+
+* api: remove dependency on proto-public, protobuf, and grpc [[GH-21780](https://github.com/hashicorp/consul/issues/21780)]
+* snapshot agent: **(Enterprise only)**  Implement Service Principal Auth for snapshot agent on azure.
+* xds: configures Envoy to load balance over all instances of an external service configured with hostnames when "envoy_dns_discovery_type" is set to "STRICT_DNS" [[GH-21655](https://github.com/hashicorp/consul/issues/21655)]
+
+## 1.20.0 (October 14, 2024)
+
+SECURITY:
+
+* Explicitly set 'Content-Type' header to mitigate XSS vulnerability. [[GH-21704](https://github.com/hashicorp/consul/issues/21704)]
+* Implement HTML sanitization for user-generated content to prevent XSS attacks in the UI. [[GH-21711](https://github.com/hashicorp/consul/issues/21711)]
+* UI: Remove codemirror linting due to package dependency [[GH-21726](https://github.com/hashicorp/consul/issues/21726)]
+* Upgrade Go to use 1.22.7. This addresses CVE 
+[CVE-2024-34155](https://nvd.nist.gov/vuln/detail/CVE-2024-34155) [[GH-21705](https://github.com/hashicorp/consul/issues/21705)]
+* Upgrade to support aws/aws-sdk-go `v1.55.5 or higher`. This resolves CVEs
+[CVE-2020-8911](https://nvd.nist.gov/vuln/detail/cve-2020-8911) and 
+[CVE-2020-8912](https://nvd.nist.gov/vuln/detail/cve-2020-8912). [[GH-21684](https://github.com/hashicorp/consul/issues/21684)]
+* ui: Pin a newer resolution of Braces [[GH-21710](https://github.com/hashicorp/consul/issues/21710)]
+* ui: Pin a newer resolution of Codemirror [[GH-21715](https://github.com/hashicorp/consul/issues/21715)]
+* ui: Pin a newer resolution of Markdown-it [[GH-21717](https://github.com/hashicorp/consul/issues/21717)]
+* ui: Pin a newer resolution of ansi-html [[GH-21735](https://github.com/hashicorp/consul/issues/21735)]
+
+FEATURES:
+
+* grafana: added the dashboards service-to-service dashboard, service dashboard, and consul dataplane dashboard [[GH-21806](https://github.com/hashicorp/consul/issues/21806)]
+* server: remove v2 tenancy, catalog, and mesh experiments [[GH-21592](https://github.com/hashicorp/consul/issues/21592)]
+
+IMPROVEMENTS:
+
+* security: upgrade ubi base image to 9.4 [[GH-21750](https://github.com/hashicorp/consul/issues/21750)]
+* connect: Add Envoy 1.31 and 1.30 to support matrix [[GH-21616](https://github.com/hashicorp/consul/issues/21616)]
+
+BUG FIXES:
+
+* jwt-provider: change dns lookup family from the default of AUTO which would prefer ipv6 to ALL if LOGICAL_DNS is used or PREFER_IPV4 if STRICT_DNS is used to gracefully handle transitions to ipv6. [[GH-21703](https://github.com/hashicorp/consul/issues/21703)]
+
 ## 1.20.0-rc1 (September 19, 2024)
 
 SECURITY:
 
 * Explicitly set 'Content-Type' header to mitigate XSS vulnerability. [[GH-21704](https://github.com/hashicorp/consul/issues/21704)]
 * Implement HTML sanitization for user-generated content to prevent XSS attacks in the UI. [[GH-21711](https://github.com/hashicorp/consul/issues/21711)]
 * UI: Remove codemirror linting due to package dependency [[GH-21726](https://github.com/hashicorp/consul/issues/21726)]
-* Upgrade Go to use 1.22.7. This addresses CVE
-  [CVE-2024-34155](https://nvd.nist.gov/vuln/detail/CVE-2024-34155) [[GH-21705](https://github.com/hashicorp/consul/issues/21705)]
+* Upgrade Go to use 1.22.7. This addresses CVE 
+[CVE-2024-34155](https://nvd.nist.gov/vuln/detail/CVE-2024-34155) [[GH-21705](https://github.com/hashicorp/consul/issues/21705)]
 * Upgrade to support aws/aws-sdk-go `v1.55.5 or higher`. This resolves CVEs
-  [CVE-2020-8911](https://nvd.nist.gov/vuln/detail/cve-2020-8911) and
-  [CVE-2020-8912](https://nvd.nist.gov/vuln/detail/cve-2020-8912). [[GH-21684](https://github.com/hashicorp/consul/issues/21684)]
+[CVE-2020-8911](https://nvd.nist.gov/vuln/detail/cve-2020-8911) and 
+[CVE-2020-8912](https://nvd.nist.gov/vuln/detail/cve-2020-8912). [[GH-21684](https://github.com/hashicorp/consul/issues/21684)]
 * ui: Pin a newer resolution of Braces [[GH-21710](https://github.com/hashicorp/consul/issues/21710)]
 * ui: Pin a newer resolution of Codemirror [[GH-21715](https://github.com/hashicorp/consul/issues/21715)]
 * ui: Pin a newer resolution of Markdown-it [[GH-21717](https://github.com/hashicorp/consul/issues/21717)]

agent/agent.go

@@ -1434,9 +1434,6 @@ func newConsulConfig(runtimeCfg *config.RuntimeConfig, logger hclog.Logger) (*co
 	cfg.GRPCTLSPort = runtimeCfg.GRPCTLSPort
 
 	cfg.Segment = runtimeCfg.SegmentName
-
-	cfg.RaftConfig.PreVoteDisabled = runtimeCfg.RaftPreVoteDisabled
-
 	if len(runtimeCfg.Segments) > 0 {
 		segments, err := segmentConfig(runtimeCfg)
 		if err != nil {

agent/config/builder.go

@@ -1073,7 +1073,6 @@ func (b *builder) build() (rt RuntimeConfig, err error) {
 		RaftSnapshotThreshold:             intVal(c.RaftSnapshotThreshold),
 		RaftSnapshotInterval:              b.durationVal("raft_snapshot_interval", c.RaftSnapshotInterval),
 		RaftTrailingLogs:                  intVal(c.RaftTrailingLogs),
-		RaftPreVoteDisabled:               boolVal(c.RaftPreVoteDisabled),
 		RaftLogStoreConfig:                b.raftLogStoreConfigVal(&c.RaftLogStore),
 		ReconnectTimeoutLAN:               b.durationVal("reconnect_timeout", c.ReconnectTimeoutLAN),
 		ReconnectTimeoutWAN:               b.durationVal("reconnect_timeout_wan", c.ReconnectTimeoutWAN),

agent/config/config.go

@@ -8,9 +8,8 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/mitchellh/mapstructure"
-
 	"github.com/hashicorp/hcl"
+	"github.com/mitchellh/mapstructure"
 
 	"github.com/hashicorp/consul/lib/decode"
 )
@@ -215,7 +214,6 @@ type Config struct {
 	RaftSnapshotThreshold            *int                `mapstructure:"raft_snapshot_threshold" json:"raft_snapshot_threshold,omitempty"`
 	RaftSnapshotInterval             *string             `mapstructure:"raft_snapshot_interval" json:"raft_snapshot_interval,omitempty"`
 	RaftTrailingLogs                 *int                `mapstructure:"raft_trailing_logs" json:"raft_trailing_logs,omitempty"`
-	RaftPreVoteDisabled              *bool               `mapstructure:"raft_prevote_disabled" json:"raft_prevote_disabled,omitempty"`
 	ReconnectTimeoutLAN              *string             `mapstructure:"reconnect_timeout" json:"reconnect_timeout,omitempty"`
 	ReconnectTimeoutWAN              *string             `mapstructure:"reconnect_timeout_wan" json:"reconnect_timeout_wan,omitempty"`
 	RejoinAfterLeave                 *bool               `mapstructure:"rejoin_after_leave" json:"rejoin_after_leave,omitempty"`

agent/config/default.go

@@ -146,7 +146,6 @@ func DefaultSource() Source {
 		raft_snapshot_threshold = ` + strconv.Itoa(int(cfg.RaftConfig.SnapshotThreshold)) + `
 		raft_snapshot_interval =  "` + cfg.RaftConfig.SnapshotInterval.String() + `"
 		raft_trailing_logs = ` + strconv.Itoa(int(cfg.RaftConfig.TrailingLogs)) + `
-		raft_prevote_disabled =  false
 		raft_logstore {
 			wal {
 				segment_size_mb = 64

agent/config/runtime.go

@@ -10,9 +10,8 @@ import (
 	"strings"
 	"time"
 
-	"golang.org/x/time/rate"
-
 	"github.com/hashicorp/go-uuid"
+	"golang.org/x/time/rate"
 
 	"github.com/hashicorp/consul/agent/cache"
 	"github.com/hashicorp/consul/agent/consul"
@@ -1005,9 +1004,6 @@ type RuntimeConfig struct {
 	// hcl: raft_trailing_logs = int
 	RaftTrailingLogs int
 
-	// hcl: raft_prevote_disabled = bool
-	RaftPreVoteDisabled bool
-
 	RaftLogStoreConfig consul.RaftLogStoreConfig
 
 	// ReconnectTimeoutLAN specifies the amount of time to wait to reconnect with

agent/config/runtime_test.go

@@ -6594,7 +6594,6 @@ func TestLoad_FullConfig(t *testing.T) {
 		RaftSnapshotThreshold:   16384,
 		RaftSnapshotInterval:    30 * time.Second,
 		RaftTrailingLogs:        83749,
-		RaftPreVoteDisabled:     false,
 		ReconnectTimeoutLAN:     23739 * time.Second,
 		ReconnectTimeoutWAN:     26694 * time.Second,
 		RequestLimitsMode:       consulrate.ModePermissive,

agent/config/testdata/TestRuntimeConfig_Sanitize.golden

@@ -71,7 +71,6 @@
     "AutopilotDisableUpgradeMigration": false,
     "AutopilotLastContactThreshold": "0s",
     "AutopilotMaxTrailingLogs": 0,
-
     "AutopilotMinQuorum": 0,
     "AutopilotRedundancyZoneTag": "",
     "AutopilotServerStabilizationTime": "0s",
@@ -300,7 +299,6 @@
     "RaftSnapshotInterval": "0s",
     "RaftSnapshotThreshold": 0,
     "RaftTrailingLogs": 0,
-    "RaftPreVoteDisabled": false,
     "ReadReplica": false,
     "ReconnectTimeoutLAN": "0s",
     "ReconnectTimeoutWAN": "0s",

agent/structs/config_entry_intentions.go

@@ -426,13 +426,15 @@ func (p *IntentionHTTPPermission) Clone() *IntentionHTTPPermission {
 }
 
 type IntentionHTTPHeaderPermission struct {
-	Name    string
-	Present bool   `json:",omitempty"`
-	Exact   string `json:",omitempty"`
-	Prefix  string `json:",omitempty"`
-	Suffix  string `json:",omitempty"`
-	Regex   string `json:",omitempty"`
-	Invert  bool   `json:",omitempty"`
+	Name       string
+	Present    bool   `json:",omitempty"`
+	Exact      string `json:",omitempty"`
+	Prefix     string `json:",omitempty"`
+	Suffix     string `json:",omitempty"`
+	Contains   string `json:",omitempty"`
+	Regex      string `json:",omitempty"`
+	Invert     bool   `json:",omitempty"`
+	IgnoreCase bool   `json:",omitempty" alias:"ignore_case"`
 }
 
 func cloneStringStringMap(m map[string]string) map[string]string {
@@ -880,8 +882,14 @@ func (e *ServiceIntentionsConfigEntry) validate(legacyWrite bool) error {
 				if hdr.Suffix != "" {
 					hdrParts++
 				}
+				if hdr.Contains != "" {
+					hdrParts++
+				}
 				if hdrParts != 1 {
-					return fmt.Errorf(errorPrefix+".Header[%d] should only contain one of Present, Exact, Prefix, Suffix, or Regex", i, j, k)
+					return fmt.Errorf(errorPrefix+".Header[%d] should only contain one of Present, Exact, Prefix, Suffix, Contains, or Regex", i, j, k)
+				}
+				if hdr.IgnoreCase && (hdr.Present || hdr.Regex != "") {
+					return fmt.Errorf(errorPrefix+".Header[%d] should set one of Exact, Prefix, Suffix, or Contains when using IgnoreCase", i, j, k)
 				}
 				permParts++
 			}