Merge branch 'dev/0.18.1' into dev/kiyoshi-kuromiya

.bumpversion.cfg

@@ -1,4 +1,3 @@
-[bumpversion]
 current_version = 0.19.0a1
 parse = (?P<major>\d+)
 	\.(?P<minor>\d+)

CHANGELOG.md

@@ -25,18 +25,49 @@ Contributors:
 - [@kingfink](https://github.com/kingfink) ([#2791](https://github.com/fishtown-analytics/dbt/pull/2791))
 - [@zmac12](https://github.com/zmac12) ([#2871](https://github.com/fishtown-analytics/dbt/pull/2817))
 
-## dbt 0.18.1 (Release TBD)
+
+## dbt 0.18.1 (October 13, 2020)
+
+## dbt 0.18.1rc1 (October 01, 2020)
+
+
+### Features
+- Added retry support for rateLimitExceeded error from BigQuery, ([#2795](https://github.com/fishtown-analytics/dbt/issues/2795), [#2796](https://github.com/fishtown-analytics/dbt/issues/2796))
+
+Contributors:
+- [@championj-foxtel](https://github.com/championj-foxtel) ([#2796](https://github.com/fishtown-analytics/dbt/issues/2796))
+
+## dbt 0.18.1b3 (September 25, 2020)
+
+
+### Feature
+- Added 'Last Modified' stat in snowflake catalog macro. Now should be available in docs. ([#2728](https://github.com/fishtown-analytics/dbt/issues/2728))
+
+### Fixes
+- `dbt compile` and `dbt run` failed with `KeyError: 'endpoint_resolver'` when threads > 1 and `method: iam` had been specified in the profiles.yaml ([#2756](https://github.com/fishtown-analytics/dbt/issues/2756), [#2766](https://github.com/fishtown-analytics/dbt/pull/2766))
+- Fix Redshift adapter to include columns from external tables when using the get_columns_in_relation macro ([#2753](https://github.com/fishtown-analytics/dbt/issues/2753), [#2754](https://github.com/fishtown-analytics/dbt/pull/2754))
+
+### Under the hood
+- Require extra `snowflake-connector-python[secure-local-storage]` on all dbt-snowflake installations ([#2779](https://github.com/fishtown-analytics/dbt/issues/2779), [#2789](https://github.com/fishtown-analytics/dbt/pull/2789))
+
+Contributors:
+- [@Mr-Nobody99](https://github.com/Mr-Nobody99) ([#2732](https://github.com/fishtown-analytics/dbt/pull/2732))
+- [@jweibel22](https://github.com/jweibel22) ([#2766](https://github.com/fishtown-analytics/dbt/pull/2766))
+- [@aiguofer](https://github.com/aiguofer) ([#2754](https://github.com/fishtown-analytics/dbt/pull/2754))
+
+
+## dbt 0.18.1b1 (September 17, 2020)
 
 ### Under the hood
 - If column config says quote, use quoting in SQL for adding a comment. ([#2539](https://github.com/fishtown-analytics/dbt/issues/2539), [#2733](https://github.com/fishtown-analytics/dbt/pull/2733))
 - Added support for running docker-based tests under Linux. ([#2739](https://github.com/fishtown-analytics/dbt/issues/2739))
 
 ### Features
 - Specify all three logging levels (`INFO`, `WARNING`, `ERROR`) in result logs for commands `test`, `seed`, `run`, `snapshot` and `source snapshot-freshness` ([#2680](https://github.com/fishtown-analytics/dbt/pull/2680), [#2723](https://github.com/fishtown-analytics/dbt/pull/2723))
-- Added "reports" ([#2730](https://github.com/fishtown-analytics/dbt/issues/2730), [#2752](https://github.com/fishtown-analytics/dbt/pull/2752))
+- Added "exposures" ([#2730](https://github.com/fishtown-analytics/dbt/issues/2730), [#2752](https://github.com/fishtown-analytics/dbt/pull/2752), [#2777](https://github.com/fishtown-analytics/dbt/issues/2777))
 
 ### Docs
-- Add Report nodes ([docs#135](https://github.com/fishtown-analytics/dbt-docs/issues/135), [docs#136](https://github.com/fishtown-analytics/dbt-docs/pull/136))
+- Add Exposure nodes ([docs#135](https://github.com/fishtown-analytics/dbt-docs/issues/135), [docs#136](https://github.com/fishtown-analytics/dbt-docs/pull/136), [docs#137](https://github.com/fishtown-analytics/dbt-docs/pull/137))
 
 Contributors:
 - [@tpilewicz](https://github.com/tpilewicz) ([#2723](https://github.com/fishtown-analytics/dbt/pull/2723))

Makefile

@@ -24,8 +24,13 @@ test-quick: .env
 # This rule creates a file named .env that is used by docker-compose for passing
 # the USER_ID and GROUP_ID arguments to the Docker image.
 .env:
+	@touch .env
+ifneq ($(OS),Windows_NT)
+ifneq ($(shell uname -s), Darwin)
 	@echo USER_ID=$(shell id -u) > .env
 	@echo GROUP_ID=$(shell id -g) >> .env
+endif
+endif
 	@time docker-compose build
 
 clean:

core/dbt/compilation.py

@@ -426,9 +426,9 @@ def link_graph(self, linker: Linker, manifest: Manifest):
             linker.add_node(source.unique_id)
         for node in manifest.nodes.values():
             self.link_node(linker, node, manifest)
-        for report in manifest.reports.values():
-            self.link_node(linker, report, manifest)
-            # linker.add_node(report.unique_id)
+        for exposure in manifest.exposures.values():
+            self.link_node(linker, exposure, manifest)
+            # linker.add_node(exposure.unique_id)
 
         cycle = linker.find_cycles()
 

core/dbt/context/providers.py

@@ -25,7 +25,7 @@
 )
 from dbt.contracts.graph.parsed import (
     ParsedMacro,
-    ParsedReport,
+    ParsedExposure,
     ParsedSeedNode,
     ParsedSourceDefinition,
 )
@@ -1325,15 +1325,15 @@ def generate_runtime_macro(
     return ctx.to_dict()
 
 
-class ReportRefResolver(BaseResolver):
+class ExposureRefResolver(BaseResolver):
     def __call__(self, *args) -> str:
         if len(args) not in (1, 2):
             ref_invalid_args(self.model, args)
         self.model.refs.append(list(args))
         return ''
 
 
-class ReportSourceResolver(BaseResolver):
+class ExposureSourceResolver(BaseResolver):
     def __call__(self, *args) -> str:
         if len(args) != 2:
             raise_compiler_error(
@@ -1344,23 +1344,23 @@ def __call__(self, *args) -> str:
         return ''
 
 
-def generate_parse_report(
-    report: ParsedReport,
+def generate_parse_exposure(
+    exposure: ParsedExposure,
     config: RuntimeConfig,
     manifest: Manifest,
     package_name: str,
 ) -> Dict[str, Any]:
     project = config.load_dependencies()[package_name]
     return {
-        'ref': ReportRefResolver(
+        'ref': ExposureRefResolver(
             None,
-            report,
+            exposure,
             project,
             manifest,
         ),
-        'source': ReportSourceResolver(
+        'source': ExposureSourceResolver(
             None,
-            report,
+            exposure,
             project,
             manifest,
         )

core/dbt/contracts/files.py

@@ -121,7 +121,7 @@ class SourceFile(JsonSchemaMixin):
     docs: List[str] = field(default_factory=list)
     macros: List[str] = field(default_factory=list)
     sources: List[str] = field(default_factory=list)
-    reports: List[str] = field(default_factory=list)
+    exposures: List[str] = field(default_factory=list)
     # any node patches in this file. The entries are names, not unique ids!
     patches: List[str] = field(default_factory=list)
     # any macro patches in this file. The entries are package, name pairs.

core/dbt/contracts/graph/compiled.py

@@ -5,7 +5,7 @@
     ParsedDataTestNode,
     ParsedHookNode,
     ParsedModelNode,
-    ParsedReport,
+    ParsedExposure,
     ParsedResource,
     ParsedRPCNode,
     ParsedSchemaTestNode,
@@ -219,8 +219,8 @@ def parsed_instance_for(compiled: CompiledNode) -> ParsedResource:
     ParsedSourceDefinition,
 ]
 
-# anything that participates in the graph: sources, reports, manifest nodes
+# anything that participates in the graph: sources, exposures, manifest nodes
 GraphMemberNode = Union[
     CompileResultNode,
-    ParsedReport,
+    ParsedExposure,
 ]

core/dbt/contracts/graph/manifest.py

@@ -1,6 +1,7 @@
 import abc
 import enum
 from dataclasses import dataclass, field
+from datetime import datetime
 from itertools import chain, islice
 from multiprocessing.synchronize import Lock
 from typing import (
@@ -15,7 +16,7 @@
 )
 from dbt.contracts.graph.parsed import (
     ParsedMacro, ParsedDocumentation, ParsedNodePatch, ParsedMacroPatch,
-    ParsedSourceDefinition, ParsedReport
+    ParsedSourceDefinition, ParsedExposure
 )
 from dbt.contracts.files import SourceFile
 from dbt.contracts.util import (
@@ -435,7 +436,7 @@ class Manifest:
     sources: MutableMapping[str, ParsedSourceDefinition]
     macros: MutableMapping[str, ParsedMacro]
     docs: MutableMapping[str, ParsedDocumentation]
-    reports: MutableMapping[str, ParsedReport]
+    exposures: MutableMapping[str, ParsedExposure]
     disabled: List[CompileResultNode]
     files: MutableMapping[str, SourceFile]
     metadata: ManifestMetadata = field(default_factory=ManifestMetadata)
@@ -460,7 +461,7 @@ def from_macros(
             sources={},
             macros=macros,
             docs={},
-            reports={},
+            exposures={},
             disabled=[],
             files=files,
         )
@@ -485,8 +486,8 @@ def sync_update_node(
             _update_into(self.nodes, new_node)
             return new_node
 
-    def update_report(self, new_report: ParsedReport):
-        _update_into(self.reports, new_report)
+    def update_exposure(self, new_exposure: ParsedExposure):
+        _update_into(self.exposures, new_exposure)
 
     def update_node(self, new_node: ManifestNode):
         _update_into(self.nodes, new_node)
@@ -729,7 +730,7 @@ def deepcopy(self):
             sources={k: _deepcopy(v) for k, v in self.sources.items()},
             macros={k: _deepcopy(v) for k, v in self.macros.items()},
             docs={k: _deepcopy(v) for k, v in self.docs.items()},
-            reports={k: _deepcopy(v) for k, v in self.reports.items()},
+            exposures={k: _deepcopy(v) for k, v in self.exposures.items()},
             disabled=[_deepcopy(n) for n in self.disabled],
             metadata=self.metadata,
             files={k: _deepcopy(v) for k, v in self.files.items()},
@@ -739,7 +740,7 @@ def writable_manifest(self):
         edge_members = list(chain(
             self.nodes.values(),
             self.sources.values(),
-            self.reports.values(),
+            self.exposures.values(),
         ))
         forward_edges, backward_edges = build_edges(edge_members)
 
@@ -748,7 +749,7 @@ def writable_manifest(self):
             sources=self.sources,
             macros=self.macros,
             docs=self.docs,
-            reports=self.reports,
+            exposures=self.exposures,
             metadata=self.metadata,
             disabled=self.disabled,
             child_map=forward_edges,
@@ -768,8 +769,8 @@ def expect(self, unique_id: str) -> GraphMemberNode:
             return self.nodes[unique_id]
         elif unique_id in self.sources:
             return self.sources[unique_id]
-        elif unique_id in self.reports:
-            return self.reports[unique_id]
+        elif unique_id in self.exposures:
+            return self.exposures[unique_id]
         else:
             # something terrible has happened
             raise dbt.exceptions.InternalException(
@@ -912,7 +913,7 @@ def __reduce_ex__(self, protocol):
             self.sources,
             self.macros,
             self.docs,
-            self.reports,
+            self.exposures,
             self.disabled,
             self.files,
             self.metadata,
@@ -947,9 +948,9 @@ class WritableManifest(ArtifactMixin):
             'The docs defined in the dbt project and its dependencies'
         ))
     )
-    reports: Mapping[UniqueID, ParsedReport] = field(
+    exposures: Mapping[UniqueID, ParsedExposure] = field(
         metadata=dict(description=(
-            'The reports defined in the dbt project and its dependencies'
+            'The exposures defined in the dbt project and its dependencies'
         ))
     )
     disabled: Optional[List[CompileResultNode]] = field(metadata=dict(

core/dbt/contracts/graph/parsed.py

@@ -23,7 +23,7 @@
     UnparsedBaseNode, FreshnessThreshold, ExternalTable,
     HasYamlMetadata, MacroArgument, UnparsedSourceDefinition,
     UnparsedSourceTableDefinition, UnparsedColumn, TestDef,
-    ReportOwner, ExposureType, MaturityType
+    ExposureOwner, ExposureType, MaturityType
 )
 from dbt.contracts.util import Replaceable, AdditionalPropertiesMixin
 from dbt.exceptions import warn_or_error
@@ -648,11 +648,11 @@ def search_name(self):
 
 
 @dataclass
-class ParsedReport(UnparsedBaseNode, HasUniqueID, HasFqn):
+class ParsedExposure(UnparsedBaseNode, HasUniqueID, HasFqn):
     name: str
     type: ExposureType
-    owner: ReportOwner
-    resource_type: NodeType = NodeType.Report
+    owner: ExposureOwner
+    resource_type: NodeType = NodeType.Exposure
     maturity: Optional[MaturityType] = None
     url: Optional[str] = None
     description: Optional[str] = None
@@ -673,25 +673,25 @@ def search_name(self):
     def tags(self):
         return []
 
-    def same_depends_on(self, old: 'ParsedReport') -> bool:
+    def same_depends_on(self, old: 'ParsedExposure') -> bool:
         return set(self.depends_on.nodes) == set(old.depends_on.nodes)
 
-    def same_description(self, old: 'ParsedReport') -> bool:
+    def same_description(self, old: 'ParsedExposure') -> bool:
         return self.description == old.description
 
-    def same_maturity(self, old: 'ParsedReport') -> bool:
+    def same_maturity(self, old: 'ParsedExposure') -> bool:
         return self.maturity == old.maturity
 
-    def same_owner(self, old: 'ParsedReport') -> bool:
+    def same_owner(self, old: 'ParsedExposure') -> bool:
         return self.owner == old.owner
 
-    def same_exposure_type(self, old: 'ParsedReport') -> bool:
+    def same_exposure_type(self, old: 'ParsedExposure') -> bool:
         return self.type == old.type
 
-    def same_url(self, old: 'ParsedReport') -> bool:
+    def same_url(self, old: 'ParsedExposure') -> bool:
         return self.url == old.url
 
-    def same_contents(self, old: Optional['ParsedReport']) -> bool:
+    def same_contents(self, old: Optional['ParsedExposure']) -> bool:
         # existing when it didn't before is a change!
         if old is None:
             return True
@@ -712,6 +712,6 @@ def same_contents(self, old: Optional['ParsedReport']) -> bool:
     ParsedDocumentation,
     ParsedMacro,
     ParsedNode,
-    ParsedReport,
+    ParsedExposure,
     ParsedSourceDefinition,
 ]

core/dbt/contracts/graph/unparsed.py

@@ -405,16 +405,16 @@ class MaturityType(StrEnum):
 
 
 @dataclass
-class ReportOwner(JsonSchemaMixin, Replaceable):
+class ExposureOwner(JsonSchemaMixin, Replaceable):
     email: str
     name: Optional[str] = None
 
 
 @dataclass
-class UnparsedReport(JsonSchemaMixin, Replaceable):
+class UnparsedExposure(JsonSchemaMixin, Replaceable):
     name: str
     type: ExposureType
-    owner: ReportOwner
+    owner: ExposureOwner
     maturity: Optional[MaturityType] = None
     url: Optional[str] = None
     description: Optional[str] = None

core/dbt/graph/cli.py

@@ -19,7 +19,7 @@
 
 INTERSECTION_DELIMITER = ','
 
-DEFAULT_INCLUDES: List[str] = ['fqn:*', 'source:*', 'report:*']
+DEFAULT_INCLUDES: List[str] = ['fqn:*', 'source:*', 'exposure:*']
 DEFAULT_EXCLUDES: List[str] = []
 DATA_TEST_SELECTOR: str = 'test_type:data'
 SCHEMA_TEST_SELECTOR: str = 'test_type:schema'

core/dbt/graph/queue.py

@@ -7,7 +7,7 @@
 import networkx as nx  # type: ignore
 
 from .graph import UniqueId
-from dbt.contracts.graph.parsed import ParsedSourceDefinition, ParsedReport
+from dbt.contracts.graph.parsed import ParsedSourceDefinition, ParsedExposure
 from dbt.contracts.graph.compiled import GraphMemberNode
 from dbt.contracts.graph.manifest import Manifest
 from dbt.node_types import NodeType
@@ -50,8 +50,8 @@ def _include_in_cost(self, node_id: UniqueId) -> bool:
         node = self.manifest.expect(node_id)
         if node.resource_type != NodeType.Model:
             return False
-        # must be a Model - tell mypy this won't be a Source or Report
-        assert not isinstance(node, (ParsedSourceDefinition, ParsedReport))
+        # must be a Model - tell mypy this won't be a Source or Exposure
+        assert not isinstance(node, (ParsedSourceDefinition, ParsedExposure))
         if node.is_ephemeral:
             return False
         return True