Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feat: Add environments command to list environments #3770

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions docs/reference/cli.md
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ Commands:
dag Render the DAG as an html file.
diff Show the diff between the local state and the...
dlt_refresh Attaches to a DLT pipeline with the option to...
environments Prints the list of SQLMesh environments with...
evaluate Evaluate a model and return a dataframe with a...
fetchdf Run a SQL query and display the results.
format Format all SQL models and audits.
Expand Down Expand Up @@ -146,6 +147,17 @@ Options:
--help Show this message and exit.
```

## environments
```
Usage: sqlmesh environments [OPTIONS]

Prints the list of SQLMesh environments with its expiration datetime.

Options:
-e, --expiry-ds Prints the expiration datetime of the environments.
--help Show this message and exit.
```

## evaluate

```
Expand Down
8 changes: 8 additions & 0 deletions docs/reference/notebook.md
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,14 @@ options:
--force, -f If set it will overwrite existing models with the new generated models from the DLT tables.
```

#### environments
```
%environments [--expiry-ds]
Prints the list of SQLMesh environments with its expiration datetime.
options:
--expiry-ds, -e If set will print the expiration datetime of the environments
```

#### fetchdf
```
%%fetchdf [df_var]
Expand Down
29 changes: 28 additions & 1 deletion sqlmesh/cli/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
from sqlmesh.core.analytics import cli_analytics
from sqlmesh.core.config import load_configs
from sqlmesh.core.context import Context
from sqlmesh.utils.date import TimeLike
from sqlmesh.utils.date import TimeLike, time_like_to_str
from sqlmesh.utils.errors import MissingDependencyError

logger = logging.getLogger(__name__)
Expand Down Expand Up @@ -966,3 +966,30 @@ def dlt_refresh(
ctx.obj.console.log_success(f"Updated SQLMesh project with models:\n{model_names}")
else:
ctx.obj.console.log_success("All SQLMesh models are up to date.")


@cli.command("environments")
@click.option(
"-e",
"--expiry-ds",
is_flag=True,
help="Prints the expiration datetime of the environments.",
default=False,
)
@click.pass_obj
@error_handler
@cli_analytics
def environments(obj: Context, expiry_ds: bool) -> None:
"""Prints the list of SQLMesh environments with its expiration datetime."""
environments = {e.name: e.expiration_ts for e in obj.state_sync.get_environments()}
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just a heads up that environment objects are pretty large and loading and parsing them all at once can be a pretty long and memory-intensive operation for big projects with a lot of environments.

A better approach could be having a specialized method in the the state_sync.py that only fetches limited fields like names.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

sure, I'll do this. Thanks :)

environment_names = list(environments.keys())
if expiry_ds:
max_width = len(max(environment_names, key=len))
print(
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we should be using the console instance here as well.

"\n".join(
f"{k:<{max_width}} {time_like_to_str(v)}" if v else f"{k:<{max_width}} no-expiry"
for k, v in environments.items()
),
)
return
print("\n".join(environment_names))
29 changes: 28 additions & 1 deletion sqlmesh/magics.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
from sqlmesh.core.dialect import format_model_expressions, parse
from sqlmesh.core.model import load_sql_based_model
from sqlmesh.core.test import ModelTestMetadata, get_all_model_tests
from sqlmesh.utils import sqlglot_dialects, yaml
from sqlmesh.utils import date, sqlglot_dialects, yaml
from sqlmesh.utils.errors import MagicError, MissingContextException, SQLMeshError

logger = logging.getLogger(__name__)
Expand Down Expand Up @@ -1007,6 +1007,33 @@ def clean(self, context: Context, line: str) -> None:
context.clear_caches()
context.console.log_success("SQLMesh cache and build artifacts cleared")

@magic_arguments()
@argument(
"--expiry-ds",
"-e",
action="store_true",
help="Prints the expiration datetime of the environments.",
)
@line_magic
@pass_sqlmesh_context
def environments(self, context: Context, line: str) -> None:
Copy link
Member

@izeigerman izeigerman Feb 2, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This looks like the exact copy-paste of the code above. It might be worse considering having a method as part of Context that prints this.

Copy link
Contributor Author

@lafirm lafirm Feb 3, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I followed the pattern used by table_name method in magics.py.

If you not mind could you please elaborate on whats expected here?

"""Prints the list of SQLMesh environments with its expiration datetime."""
args = parse_argstring(self.environments, line)
environments = {e.name: e.expiration_ts for e in context.state_sync.get_environments()}
environment_names = list(environments.keys())
if args.expiry_ds:
max_width = len(max(environment_names, key=len))
context.console.log_status_update(
"\n".join(
f"{k:<{max_width}} {date.time_like_to_str(v)}"
if v
else f"{k:<{max_width}} no-expiry"
for k, v in environments.items()
),
)
return
context.console.log_status_update("\n".join(environment_names))


def register_magics() -> None:
try:
Expand Down
67 changes: 66 additions & 1 deletion tests/cli/test_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
from sqlmesh.cli.main import cli
from sqlmesh.core.context import Context
from sqlmesh.integrations.dlt import generate_dlt_models
from sqlmesh.utils.date import yesterday_ds
from sqlmesh.utils.date import now_ds, time_like_to_str, timedelta, to_datetime, yesterday_ds

FREEZE_TIME = "2023-01-01 00:00:00 UTC"

Expand Down Expand Up @@ -971,3 +971,68 @@ def test_init_project_dialects(runner, tmp_path):
assert config == f"{config_start}{expected_config}{config_end}"

remove(tmp_path / "config.yaml")


def test_environments(runner, tmp_path):
create_example_project(tmp_path)

# create dev environment and backfill
runner.invoke(
cli,
[
"--log-file-dir",
tmp_path,
"--paths",
tmp_path,
"plan",
"dev",
"--no-prompts",
"--auto-apply",
],
)

# # create dev2 environment from dev environment
# # Input: `y` to apply and virtual update
runner.invoke(
cli,
[
"--log-file-dir",
tmp_path,
"--paths",
tmp_path,
"plan",
"dev2",
"--create-from",
"dev",
"--include-unmodified",
],
input="y\n",
)

result = runner.invoke(
cli,
[
"--log-file-dir",
tmp_path,
"--paths",
tmp_path,
"environments",
],
)
assert result.exit_code == 0
assert result.output == "dev\ndev2\n"

result = runner.invoke(
cli,
[
"--log-file-dir",
tmp_path,
"--paths",
tmp_path,
"environments",
"--expiry-ds",
],
)
assert result.exit_code == 0
ttl = time_like_to_str(to_datetime(now_ds()) + timedelta(days=7))
assert result.output == f"dev {ttl}\ndev2 {ttl}\n"