From c761e3f8a1d0385583ae43a0e5c1261c1f5bf7e0 Mon Sep 17 00:00:00 2001 From: Mathieu Gabelle Date: Thu, 2 Jan 2025 10:33:56 +0100 Subject: [PATCH] refactor: move dbt cli to dynamic properties --- src/main/java/io/kestra/plugin/dbt/cli/AbstractDbt.java | 6 ++++-- src/main/java/io/kestra/plugin/dbt/cli/DbtCLI.java | 6 +++--- src/main/java/io/kestra/plugin/dbt/cli/Setup.java | 6 +++--- src/test/java/io/kestra/plugin/dbt/cli/DbtCLITest.java | 6 +++--- 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/main/java/io/kestra/plugin/dbt/cli/AbstractDbt.java b/src/main/java/io/kestra/plugin/dbt/cli/AbstractDbt.java index 3110d71..7a2368e 100644 --- a/src/main/java/io/kestra/plugin/dbt/cli/AbstractDbt.java +++ b/src/main/java/io/kestra/plugin/dbt/cli/AbstractDbt.java @@ -134,17 +134,19 @@ public void setDockerOptions(Property dockerOptions) { private Object inputFiles; - private List outputFiles; + private Property> outputFiles; protected abstract java.util.List dbtCommands(RunContext runContext) throws IllegalVariableEvaluationException; @Override public ScriptOutput run(RunContext runContext) throws Exception { + var renderedOutputFiles = runContext.render(this.outputFiles).asList(String.class); + CommandsWrapper commandsWrapper = new CommandsWrapper(runContext) .withEnv(runContext.render(this.getEnv()).asMap(String.class, String.class)) .withNamespaceFiles(namespaceFiles) .withInputFiles(inputFiles) - .withOutputFiles(outputFiles) + .withOutputFiles(renderedOutputFiles.isEmpty() ? null : renderedOutputFiles) .withRunnerType(runContext.render(this.getRunner()).as(RunnerType.class).orElse(null)) .withDockerOptions(runContext.render(this.getDocker()).as(DockerOptions.class).orElse(null)) .withContainerImage(runContext.render(this.getContainerImage()).as(String.class).orElseThrow()) diff --git a/src/main/java/io/kestra/plugin/dbt/cli/DbtCLI.java b/src/main/java/io/kestra/plugin/dbt/cli/DbtCLI.java index 2038c08..ce1d866 100644 --- a/src/main/java/io/kestra/plugin/dbt/cli/DbtCLI.java +++ b/src/main/java/io/kestra/plugin/dbt/cli/DbtCLI.java @@ -320,7 +320,7 @@ public class DbtCLI extends AbstractExecScript { .build(); @Builder.Default - protected String containerImage = DEFAULT_IMAGE; + protected Property containerImage = Property.of(DEFAULT_IMAGE); @Schema( title = "Store manifest.", @@ -345,7 +345,7 @@ protected DockerOptions injectDefaults(DockerOptions original) { var builder = original.toBuilder(); if (original.getImage() == null) { - builder.image(this.getContainerImage()); + builder.image(this.getContainerImage().toString()); } if (original.getEntryPoint() == null) { builder.entryPoint(new ArrayList<>()); @@ -399,7 +399,7 @@ public void accept(String line, Boolean isStdErr) { //Create and run commands List commandsArgs = ScriptService.scriptCommands( this.interpreter, - this.getBeforeCommandsWithOptions(), + this.getBeforeCommandsWithOptions(runContext), runContext.render(this.commands).stream().map(command -> command.startsWith("dbt") ? command.concat(" --log-format json") : command).toList() ); diff --git a/src/main/java/io/kestra/plugin/dbt/cli/Setup.java b/src/main/java/io/kestra/plugin/dbt/cli/Setup.java index 08f0d5c..6af8e02 100644 --- a/src/main/java/io/kestra/plugin/dbt/cli/Setup.java +++ b/src/main/java/io/kestra/plugin/dbt/cli/Setup.java @@ -147,7 +147,7 @@ public class Setup extends AbstractExecScript implements RunnableTask containerImage = Property.of(DEFAULT_IMAGE); @Schema(title = "Deprecated, use the `docker` property instead", deprecated = true) @Deprecated @@ -163,7 +163,7 @@ public void setDockerOptions(DockerOptions dockerOptions) { protected DockerOptions injectDefaults(DockerOptions original) { var builder = original.toBuilder(); if (original.getImage() == null) { - builder.image(this.getContainerImage()); + builder.image(this.getContainerImage().toString()); } return builder.build(); @@ -197,7 +197,7 @@ public ScriptOutput run(RunContext runContext) throws Exception { List commandsArgs = ScriptService.scriptCommands( this.interpreter, - this.getBeforeCommandsWithOptions(), + this.getBeforeCommandsWithOptions(runContext), commands ); diff --git a/src/test/java/io/kestra/plugin/dbt/cli/DbtCLITest.java b/src/test/java/io/kestra/plugin/dbt/cli/DbtCLITest.java index dde833a..92cbe4e 100644 --- a/src/test/java/io/kestra/plugin/dbt/cli/DbtCLITest.java +++ b/src/test/java/io/kestra/plugin/dbt/cli/DbtCLITest.java @@ -75,7 +75,7 @@ void run() throws Exception { .type(DbtCLI.class.getName()) .profiles(Property.of(PROFILES) ) - .containerImage("ghcr.io/kestra-io/dbt-bigquery:latest") + .containerImage(new Property<>("ghcr.io/kestra-io/dbt-bigquery:latest")) .commands(List.of("dbt build")) .build(); @@ -97,7 +97,7 @@ void testDbtCliWithStoreManifest_manifestShouldBePresentInKvStore() throws Excep .type(DbtCLI.class.getName()) .profiles(Property.of(PROFILES) ) - .containerImage("ghcr.io/kestra-io/dbt-bigquery:latest") + .containerImage(new Property<>("ghcr.io/kestra-io/dbt-bigquery:latest")) .commands(List.of("dbt build")) .storeManifest( DbtCLI.KvStoreManifest.builder() @@ -130,7 +130,7 @@ void testDbtWithLoadManifest_manifestShouldBeLoadedFromKvStore() throws Exceptio .type(DbtCLI.class.getName()) .profiles(Property.of(PROFILES)) .projectDir(Property.of("unit-kestra")) - .containerImage("ghcr.io/kestra-io/dbt-bigquery:latest") + .containerImage(new Property<>("ghcr.io/kestra-io/dbt-bigquery:latest")) .commands(List.of("dbt build --project-dir unit-kestra")) .loadManifest( DbtCLI.KvStoreManifest.builder()