Merge branch 'main' into test-ut-input-ephemeral-with-sql-format

.bumpversion.cfg

@@ -35,5 +35,3 @@ first_value = 1
 [bumpversion:file:core/setup.py]
 
 [bumpversion:file:core/dbt/version.py]
-
-[bumpversion:file:docker/Dockerfile]

.changes/unreleased/Dependencies-20240410-183321.yaml

@@ -0,0 +1,6 @@
+kind: "Dependencies"
+body: "Bump black from 23.3.0 to >=24.3.0,<25.0"
+time: 2024-04-10T18:33:21.00000Z
+custom:
+  Author: dependabot[bot]
+  PR: 8074

.changes/unreleased/Dependencies-20240415-202426.yaml

@@ -0,0 +1,6 @@
+kind: Dependencies
+body: Bump sqlparse to >=0.5.0, <0.6.0
+time: 2024-04-15T20:24:26.768707-05:00
+custom:
+  Author: emmyoop
+  Issue: "9951"

.changes/unreleased/Features-20240307-153622.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Support scrubbing secret vars
+time: 2024-03-07T15:36:22.754627+01:00
+custom:
+  Author: nielspardon
+  Issue: "7247"

.changes/unreleased/Under the Hood-20240416-150030.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Add a test for semantic manifest and move test fixtures needed for it
+time: 2024-04-16T15:00:30.614286-07:00
+custom:
+  Author: ChenyuLInx
+  Issue: "9665"

.github/workflows/nightly-release.yml

@@ -20,6 +20,7 @@ on:
 
 permissions:
   contents: write # this is the permission that allows creating a new release
+  packages: write # this is the permission that allows Docker release
 
 defaults:
   run:

.github/workflows/release.yml

@@ -39,6 +39,11 @@ on:
         type: boolean
         default: false
         required: false
+      only_docker:
+        description: "Only release Docker image, skip GitHub & PyPI"
+        type: boolean
+        default: false
+        required: false
   workflow_call:
     inputs:
       target_branch:
@@ -81,6 +86,7 @@ jobs:
           echo The release version number:         ${{ inputs.version_number }}
           echo Test run:                           ${{ inputs.test_run }}
           echo Nightly release:                    ${{ inputs.nightly_release }}
+          echo Only Docker:                        ${{ inputs.only_docker }}
 
       - name: "Checkout target branch"
         uses: actions/checkout@v4
@@ -99,6 +105,7 @@ jobs:
   bump-version-generate-changelog:
     name: Bump package version, Generate changelog
     needs: [job-setup]
+    if: ${{ !inputs.only_docker }}
 
     uses: dbt-labs/dbt-release/.github/workflows/release-prep.yml@main
 
@@ -114,7 +121,7 @@ jobs:
 
   log-outputs-bump-version-generate-changelog:
     name: "[Log output] Bump package version, Generate changelog"
-    if: ${{ !failure() && !cancelled() }}
+    if: ${{ !failure() && !cancelled() && !inputs.only_docker }}
 
     needs: [bump-version-generate-changelog]
 
@@ -128,7 +135,7 @@ jobs:
 
   build-test-package:
     name: Build, Test, Package
-    if: ${{ !failure() && !cancelled() }}
+    if: ${{ !failure() && !cancelled() && !inputs.only_docker }}
     needs: [job-setup, bump-version-generate-changelog]
 
     uses: dbt-labs/dbt-release/.github/workflows/build.yml@main
@@ -149,7 +156,7 @@ jobs:
 
   github-release:
     name: GitHub Release
-    if: ${{ !failure() && !cancelled() }}
+    if: ${{ !failure() && !cancelled() && !inputs.only_docker }}
 
     needs: [bump-version-generate-changelog, build-test-package]
 
@@ -180,6 +187,7 @@ jobs:
     # dbt-postgres exists within dbt-core for versions 1.7 and earlier but is a separate package for 1.8 and later.
     # determine if we need to release dbt-core or both dbt-core and dbt-postgres
     name: Determine Docker Package
+    if: ${{ !failure() && !cancelled() }}
     runs-on: ubuntu-latest
     needs: [pypi-release]
     outputs:
@@ -204,6 +212,10 @@ jobs:
   docker-release:
     name: "Docker Release for ${{ matrix.package }}"
     needs: [determine-docker-package]
+    # We cannot release to docker on a test run because it uses the tag in GitHub as
+    # what we need to release but draft releases don't actually tag the commit so it
+    # finds nothing to release
+    if: ${{ !failure() && !cancelled() && (!inputs.test_run || inputs.only_docker) }}
     strategy:
       matrix: ${{fromJson(needs.determine-docker-package.outputs.matrix)}}
 

core/dbt/artifacts/resources/v1/semantic_model.py

@@ -42,7 +42,7 @@ class NodeRelation(dbtClassMixin):
     alias: str
     schema_name: str  # TODO: Could this be called simply "schema" so we could reuse StateRelation?
     database: Optional[str] = None
-    relation_name: Optional[str] = None
+    relation_name: Optional[str] = ""
 
 
 # ====================================

core/dbt/artifacts/schemas/run/v5/run.py

@@ -1,9 +1,11 @@
 import threading
 from typing import Any, Optional, Iterable, Tuple, Sequence, Dict, TYPE_CHECKING
+import copy
 from dataclasses import dataclass, field
 from datetime import datetime
 
 
+from dbt.constants import SECRET_ENV_PREFIX
 from dbt.artifacts.resources import CompiledResource
 from dbt.artifacts.schemas.base import (
     BaseArtifactMetadata,
@@ -19,6 +21,7 @@
     ExecutionResult,
 )
 from dbt_common.clients.system import write_json
+from dbt.exceptions import scrub_secrets
 
 
 if TYPE_CHECKING:
@@ -123,7 +126,26 @@ def from_execution_results(
             dbt_schema_version=str(cls.dbt_schema_version),
             generated_at=generated_at,
         )
-        return cls(metadata=meta, results=processed_results, elapsed_time=elapsed_time, args=args)
+
+        secret_vars = [
+            v for k, v in args["vars"].items() if k.startswith(SECRET_ENV_PREFIX) and v.strip()
+        ]
+
+        scrubbed_args = copy.deepcopy(args)
+
+        # scrub secrets in invocation command
+        scrubbed_args["invocation_command"] = scrub_secrets(
+            scrubbed_args["invocation_command"], secret_vars
+        )
+
+        # scrub secrets in vars dict
+        scrubbed_args["vars"] = {
+            k: scrub_secrets(v, secret_vars) for k, v in scrubbed_args["vars"].items()
+        }
+
+        return cls(
+            metadata=meta, results=processed_results, elapsed_time=elapsed_time, args=scrubbed_args
+        )
 
     @classmethod
     def compatible_previous_versions(cls) -> Iterable[Tuple[str, int]]:

core/dbt/exceptions.py

@@ -17,6 +17,8 @@
 
 from dbt_common.dataclass_schema import ValidationError
 
+from dbt.constants import SECRET_ENV_PREFIX
+
 
 if TYPE_CHECKING:
     import agate
@@ -333,7 +335,10 @@ def get_message(self) -> str:
         pretty_vars = json.dumps(dct, sort_keys=True, indent=4)
 
         msg = f"Required var '{self.var_name}' not found in config:\nVars supplied to {node_name} = {pretty_vars}"
-        return msg
+        return scrub_secrets(msg, self.var_secrets())
+
+    def var_secrets(self) -> List[str]:
+        return [v for k, v in self.merged.items() if k.startswith(SECRET_ENV_PREFIX) and v.strip()]
 
 
 class PackageNotFoundForMacroError(CompilationError):

core/dbt/parser/manifest.py

@@ -44,6 +44,7 @@
     MANIFEST_FILE_NAME,
     PARTIAL_PARSE_FILE_NAME,
     SEMANTIC_MANIFEST_FILE_NAME,
+    SECRET_ENV_PREFIX,
 )
 from dbt_common.helper_types import PathSet
 from dbt_common.events.functions import fire_event, get_invocation_id, warn_or_error
@@ -116,6 +117,7 @@
     TargetNotFoundError,
     AmbiguousAliasError,
     InvalidAccessTypeError,
+    scrub_secrets,
 )
 from dbt.parser.base import Parser
 from dbt.parser.analysis import AnalysisParser
@@ -989,6 +991,9 @@ def build_manifest_state_check(self):
         # of env_vars, that would need to change.
         # We are using the parsed cli_vars instead of config.args.vars, in order
         # to sort them and avoid reparsing because of ordering issues.
+        secret_vars = [
+            v for k, v in config.cli_vars.items() if k.startswith(SECRET_ENV_PREFIX) and v.strip()
+        ]
         stringified_cli_vars = pprint.pformat(config.cli_vars)
         vars_hash = FileHash.from_contents(
             "\x00".join(
@@ -1003,7 +1008,7 @@ def build_manifest_state_check(self):
         fire_event(
             StateCheckVarsHash(
                 checksum=vars_hash.checksum,
-                vars=stringified_cli_vars,
+                vars=scrub_secrets(stringified_cli_vars, secret_vars),
                 profile=config.args.profile,
                 target=config.args.target,
                 version=__version__,

core/setup.py

@@ -67,7 +67,7 @@
         # These packages are major-version-0. Keep upper bounds on upcoming minor versions (which could have breaking changes)
         # and check compatibility / bump in each new minor version of dbt-core.
         "pathspec>=0.9,<0.13",
-        "sqlparse>=0.2.3,<0.5",
+        "sqlparse>=0.5.0,<0.6.0",
         # ----
         # These are major-version-0 packages also maintained by dbt-labs.
         # Accept patches but avoid automatically updating past a set minor version range.

dev-requirements.txt

@@ -2,7 +2,7 @@ git+https://github.com/dbt-labs/dbt-adapters.git
 git+https://github.com/dbt-labs/dbt-adapters.git@main#subdirectory=dbt-tests-adapter
 git+https://github.com/dbt-labs/dbt-common.git@main
 git+https://github.com/dbt-labs/dbt-postgres.git@main
-black==23.3.0
+black>=24.3.0,<25.0
 bumpversion
 ddtrace==2.3.0
 docutils