Skip to content

Commit

Permalink
Allow snapshots to be defined with YAML only.
Browse files Browse the repository at this point in the history
  • Loading branch information
peterallenwebb committed Sep 23, 2024
1 parent db69473 commit 7bf392e
Show file tree
Hide file tree
Showing 4 changed files with 91 additions and 0 deletions.
6 changes: 6 additions & 0 deletions .changes/unreleased/Features-20240920-110447.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
kind: Features
body: Allow snapshots to be defined in YAML.
time: 2024-09-20T11:04:47.703117-04:00
custom:
Author: peterallenwebb
Issue: "10246"
50 changes: 50 additions & 0 deletions core/dbt/parser/schemas.py
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,7 @@ def parse_file(self, block: FileBlock, dct: Optional[Dict] = None) -> None:

# PatchParser.parse()
if "snapshots" in dct:
self._add_yaml_snapshot_nodes_to_manifest(dct["snapshots"], block)
snapshot_parse_result = TestablePatchParser(self, yaml_block, "snapshots").parse()
for test_block in snapshot_parse_result.test_blocks:
self.generic_test_parser.parse_tests(test_block)
Expand Down Expand Up @@ -265,6 +266,55 @@ def parse_file(self, block: FileBlock, dct: Optional[Dict] = None) -> None:
saved_query_parser = SavedQueryParser(self, yaml_block)
saved_query_parser.parse()

def _add_yaml_snapshot_nodes_to_manifest(
self, snapshots: List[Dict[str, Any]], block: FileBlock
) -> None:
"""We support the creation of simple snapshots in yaml, without an
accompanying SQL definition. For such snapshots, the user must supply
a 'relation' property to indicate the target of the snapshot. This
function looks for such snapshots and adds a node to manifest for each
one we find, since they were not added during SQL parsing."""

rebuild_refs = False
for snapshot in snapshots:
if "relation" in snapshot:
from dbt.parser import SnapshotParser

if "name" not in snapshot:
raise ParsingError("A snapshot must define the 'name' property. ")

Check warning on line 284 in core/dbt/parser/schemas.py

View check run for this annotation

Codecov / codecov/patch

core/dbt/parser/schemas.py#L284

Added line #L284 was not covered by tests

# Reuse the logic of SnapshotParser as far as possible to create
# a new node we can add to the manifest.
parser = SnapshotParser(self.project, self.manifest, self.root_project)
fqn = parser.get_fqn_prefix(block.path.relative_path)
fqn.append(snapshot["name"])
snapshot_node = parser._create_parsetime_node(
block,
self.get_compiled_path(block),
parser.initial_config(fqn),
fqn,
snapshot["name"],
)

# Parse the expected ref() or source() expression given by
# 'relation' so that we know what we are snapshotting.
source_or_ref = statically_parse_ref_or_source(snapshot["relation"])
if isinstance(source_or_ref, RefArgs):
snapshot_node.refs.append(source_or_ref)
else:
snapshot_node.sources.append(source_or_ref)

Check warning on line 305 in core/dbt/parser/schemas.py

View check run for this annotation

Codecov / codecov/patch

core/dbt/parser/schemas.py#L305

Added line #L305 was not covered by tests

# Implement the snapshot SQL as a simple select *
snapshot_node.raw_code = "select * from {{ " + snapshot["relation"] + " }}"

# Add our new node to the manifest, and note that ref lookup collections
# will need to be rebuilt.
self.manifest.add_node_nofile(snapshot_node)
rebuild_refs = True

if rebuild_refs:
self.manifest.rebuild_ref_lookup()


Parsed = TypeVar("Parsed", UnpatchedSourceDefinition, ParsedNodePatch, ParsedMacroPatch)
NodeTarget = TypeVar("NodeTarget", UnparsedNodeUpdate, UnparsedAnalysisUpdate, UnparsedModelUpdate)
Expand Down
13 changes: 13 additions & 0 deletions tests/functional/snapshots/fixtures.py
Original file line number Diff line number Diff line change
Expand Up @@ -291,6 +291,19 @@
{% endsnapshot %}
"""

snapshots_pg__snapshot_yml = """
version: 2
snapshots:
- name: snapshot_actual
relation: "ref('seed')"
config:
unique_key: "id || '-' || first_name"
strategy: timestamp
updated_at: updated_at
meta:
owner: 'a_owner'
"""

snapshots_pg__snapshot_no_target_schema_sql = """
{% snapshot snapshot_actual %}
Expand Down
22 changes: 22 additions & 0 deletions tests/functional/snapshots/test_basic_snapshot.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
seeds__seed_newcol_csv,
snapshots_pg__snapshot_no_target_schema_sql,
snapshots_pg__snapshot_sql,
snapshots_pg__snapshot_yml,
snapshots_pg_custom__snapshot_sql,
snapshots_pg_custom_namespaced__snapshot_sql,
)
Expand Down Expand Up @@ -372,3 +373,24 @@ def test_updated_at_snapshot(self, project):
class TestRefUpdatedAtCheckCols(UpdatedAtCheckCols):
def test_updated_at_ref(self, project):
ref_setup(project, num_snapshot_models=2)


class BasicYaml(Basic):
@pytest.fixture(scope="class")
def snapshots(self):
"""Overrides the same function in Basic to use the YAML method of
defining a snapshot."""
return {"snapshot.yml": snapshots_pg__snapshot_yml}

@pytest.fixture(scope="class")
def models(self):
"""Overrides the same function in Basic to use a modified version of
schema.yml without snapshot config."""
return {
"ref_snapshot.sql": models__ref_snapshot_sql,
}


class TestBasicSnapshotYaml(BasicYaml):
def test_basic_snapshot_yaml(self, project):
snapshot_setup(project, num_snapshot_models=1)

0 comments on commit 7bf392e

Please sign in to comment.