-
Notifications
You must be signed in to change notification settings - Fork 114
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Update DataCatalog private methods with DataCatalog 2.0 API public methods #2274
base: main
Are you sure you want to change the base?
Changes from 3 commits
fbdd26b
43c5248
69c8480
7941f82
b1e87a3
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -6,6 +6,13 @@ | |
|
||
from kedro.io import DataCatalog | ||
|
||
try: | ||
from kedro.io import KedroDataCatalog | ||
|
||
IS_DATACATALOG_2 = True | ||
except ImportError: | ||
IS_DATACATALOG_2 = False | ||
|
||
try: | ||
# kedro 0.18.11 onwards | ||
from kedro.io.core import DatasetError | ||
|
@@ -91,7 +98,10 @@ def resolve_dataset_factory_patterns( | |
|
||
for dataset_name in datasets: | ||
try: | ||
catalog._get_dataset(dataset_name, suggest=False) | ||
if IS_DATACATALOG_2 and isinstance(catalog, KedroDataCatalog): | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Not regarding this PR but I remember Elena mentioned |
||
catalog.get(dataset_name) | ||
else: | ||
catalog._get_dataset(dataset_name, suggest=False) | ||
except Exception: # noqa: BLE001 # pragma: no cover | ||
continue | ||
|
||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -5,6 +5,14 @@ | |
from typing import TYPE_CHECKING, Dict, Optional | ||
|
||
from kedro.io import DataCatalog | ||
|
||
try: | ||
from kedro.io import KedroDataCatalog | ||
|
||
IS_DATACATALOG_2 = True | ||
except ImportError: | ||
IS_DATACATALOG_2 = False | ||
|
||
from packaging.version import parse | ||
|
||
from kedro_viz.constants import KEDRO_VERSION | ||
|
@@ -78,11 +86,14 @@ def layers_mapping(self): # noqa: PLR0912 | |
|
||
self._layers_mapping = {} | ||
|
||
# Temporary try/except block so the Kedro develop branch can work with Viz. | ||
try: | ||
datasets = self._catalog._data_sets | ||
except Exception: # noqa: BLE001 # pragma: no cover | ||
datasets = self._catalog._datasets | ||
if IS_DATACATALOG_2 and isinstance(self._catalog, KedroDataCatalog): | ||
datasets = self._catalog.list() | ||
else: | ||
# Temporary try/except block so the Kedro develop branch can work with Viz. | ||
try: | ||
datasets = self._catalog._data_sets | ||
except Exception: # noqa: BLE001 # pragma: no cover | ||
datasets = self._catalog._datasets | ||
|
||
# Support for Kedro 0.18.x | ||
if KEDRO_VERSION < parse("0.19.0"): # pragma: no cover | ||
|
@@ -99,8 +110,11 @@ def layers_mapping(self): # noqa: PLR0912 | |
self._layers_mapping[dataset_name] = layer | ||
|
||
for dataset_name in datasets: | ||
dataset = self._catalog._get_dataset(dataset_name) | ||
metadata = getattr(dataset, "metadata", None) | ||
if IS_DATACATALOG_2 and isinstance(self._catalog, KedroDataCatalog): | ||
dataset = self._catalog.get(dataset_name) | ||
else: | ||
dataset = self._catalog._get_dataset(dataset_name) | ||
metadata = getattr(dataset, "metadata", None) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. how are we handling metadata in 2.0 ? |
||
if not metadata: | ||
continue | ||
try: | ||
|
@@ -121,11 +135,11 @@ def layers_mapping(self): # noqa: PLR0912 | |
def get_dataset(self, dataset_name: str) -> Optional["AbstractDataset"]: | ||
dataset_obj: Optional["AbstractDataset"] | ||
try: | ||
# Kedro 0.18.1 introduced the `suggest` argument to disable the expensive | ||
# fuzzy-matching process. | ||
if KEDRO_VERSION >= parse("0.18.1"): | ||
if IS_DATACATALOG_2 and isinstance(self._catalog, KedroDataCatalog): | ||
dataset_obj = self._catalog.get(dataset_name) | ||
elif KEDRO_VERSION >= parse("0.18.1"): | ||
dataset_obj = self._catalog._get_dataset(dataset_name, suggest=False) | ||
else: # pragma: no cover | ||
else: | ||
dataset_obj = self._catalog._get_dataset(dataset_name) | ||
except DatasetNotFoundError: | ||
dataset_obj = MemoryDataset() | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
does it make sense to import this only as a fallback if there's an importError when we are in an older kedro version?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I think we still need
DataCatalog
as the “baseline” import so our code is backward compatible with older Kedro (which doesn’t haveKedroDataCatalog
)There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
should the line 7 be inside except ImportError at line 13 ? Normally that is how we were doing bc before. Wouldn't this raise error if DataCatalog is not available in future ?