From 6e01c9807c36a161d75ebd6727c3bf11e24ca43b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pa=C5=82ka?= Date: Wed, 25 Sep 2024 10:23:51 +0200 Subject: [PATCH 1/5] Pass plugin discoverer output via a temp file --- .../LanguagePluginTest.test.scala | 2 +- language-plugin/bootstrap/PulumiPlugins.scala | 22 +++++++++++++++++-- language-plugin/bootstrap/project.scala | 2 +- .../executors/executor.go | 7 ++++++ .../executors/executor_gradle.go | 10 ++++----- .../executors/executor_jar.go | 7 +++--- .../executors/executor_maven.go | 8 +++---- .../executors/executor_sbt.go | 15 ++++++------- .../executors/executor_scala_cli.go | 7 +++--- language-plugin/pulumi-language-scala/main.go | 16 ++++++++++++-- 10 files changed, 66 insertions(+), 30 deletions(-) diff --git a/integration-tests/LanguagePluginTest.test.scala b/integration-tests/LanguagePluginTest.test.scala index 025fd74c..6df7b45c 100644 --- a/integration-tests/LanguagePluginTest.test.scala +++ b/integration-tests/LanguagePluginTest.test.scala @@ -283,7 +283,7 @@ class LanguagePluginTest extends munit.FunSuite { projectFiles = Map("build.sbt" -> sbtBuildFile) ), teardown = pulumi.fixture.teardown - ).test("sbt".tag(LocalOnly)) { ctx => + ).test("sbt") { ctx => testExecutor(ctx) } diff --git a/language-plugin/bootstrap/PulumiPlugins.scala b/language-plugin/bootstrap/PulumiPlugins.scala index 398fb900..a99017fb 100644 --- a/language-plugin/bootstrap/PulumiPlugins.scala +++ b/language-plugin/bootstrap/PulumiPlugins.scala @@ -2,13 +2,20 @@ package besom.bootstrap import scala.util.Using import scala.jdk.CollectionConverters.* +import java.io.{File, PrintWriter} import io.github.classgraph.ClassGraph import besom.json.* object PulumiPluginsDiscoverer: def main(args: Array[String]): Unit = - val foundPlugins = pluginsFromClasspath - print(JsArray(foundPlugins.values.toVector).compactPrint) + args.toList match + case Nil => + print(pluginsJsonText) + case "--output-file" :: outputFilePath :: Nil => + writeToFile(path = outputFilePath, text = pluginsJsonText) + case _ => + Console.err.println(s"PulumiPluginDiscoverer: Wrong main arguments: ${args.mkString(",")}") + sys.exit(1) private val PluginRegex = "^(besom/(.+))/plugin.json$".r @@ -27,3 +34,14 @@ object PulumiPluginsDiscoverer: new ClassGraph() .filterClasspathElements(_ => true) // todo exclude garbage .scan() + + private def pluginsJsonText = + val foundPlugins = pluginsFromClasspath + JsArray(foundPlugins.values.toVector).compactPrint + + private def writeToFile(path: String, text: String): Unit = + val file = new File(path) + val writer = new PrintWriter(file) + + writer.write(text) + writer.close() diff --git a/language-plugin/bootstrap/project.scala b/language-plugin/bootstrap/project.scala index a922844a..f098fd47 100644 --- a/language-plugin/bootstrap/project.scala +++ b/language-plugin/bootstrap/project.scala @@ -2,6 +2,6 @@ //> using options -java-output-version:11 //> using dep org.virtuslab::besom-json:0.4.0-SNAPSHOT -//> using dep io.github.classgraph:classgraph:4.8.165 +//> using dep io.github.classgraph:classgraph:4.8.176 diff --git a/language-plugin/pulumi-language-scala/executors/executor.go b/language-plugin/pulumi-language-scala/executors/executor.go index e7fb7d52..27d87677 100644 --- a/language-plugin/pulumi-language-scala/executors/executor.go +++ b/language-plugin/pulumi-language-scala/executors/executor.go @@ -4,6 +4,7 @@ package executors import ( "fmt" + "path" "github.com/virtuslab/besom/language-host/fsys" ) @@ -94,3 +95,9 @@ func (c combinedScalaExecutorFactory) NewScalaExecutor(opts ScalaExecutorOptions func combineScalaExecutorFactories(variations ...scalaExecutorFactory) scalaExecutorFactory { return combinedScalaExecutorFactory(variations) } + +const PluginDiscovererOutputFileName = ".besom-pulumi-plugins.json" + +func PluginDiscovererOutputFilePath(projectRoot fsys.ParentFS) string { + return path.Join(projectRoot.Path(), PluginDiscovererOutputFileName) +} diff --git a/language-plugin/pulumi-language-scala/executors/executor_gradle.go b/language-plugin/pulumi-language-scala/executors/executor_gradle.go index 33068b52..adf64be1 100644 --- a/language-plugin/pulumi-language-scala/executors/executor_gradle.go +++ b/language-plugin/pulumi-language-scala/executors/executor_gradle.go @@ -35,7 +35,8 @@ func (g gradle) NewScalaExecutor(opts ScalaExecutorOptions) (*ScalaExecutor, err if err != nil { return nil, err } - executor, err := g.newGradleExecutor(gradleRoot, cmd, subproject, opts.BootstrapLibJarPath) + pluginDiscovererOutputPath := PluginDiscovererOutputFilePath(opts.WD) + executor, err := g.newGradleExecutor(gradleRoot, cmd, subproject, opts.BootstrapLibJarPath, pluginDiscovererOutputPath) if err != nil { return nil, err } @@ -104,7 +105,7 @@ func (gradle) isGradleProject(dir fs.FS, opts ScalaExecutorOptions) (bool, error return false, nil } -func (g gradle) newGradleExecutor(gradleRoot fsys.ParentFS, cmd, subproject string, bootstrapLibJarPath string) (*ScalaExecutor, error) { +func (g gradle) newGradleExecutor(gradleRoot fsys.ParentFS, cmd, subproject string, bootstrapLibJarPath string, pluginDiscovererOutputPath string) (*ScalaExecutor, error) { se := &ScalaExecutor{ Name: "gradle", Cmd: cmd, @@ -112,13 +113,10 @@ func (g gradle) newGradleExecutor(gradleRoot fsys.ParentFS, cmd, subproject stri BuildArgs: []string{g.prefix(subproject, "build"), "--console=plain"}, RunArgs: []string{g.prefix(subproject, "run"), "--console=plain"}, PluginArgs: []string{ - /* STDOUT needs to be clean of gradle output, - because we expect a JSON with plugin - results */ - "-q", // must go first due to a bug https://github.com/gradle/gradle/issues/5098 g.prefix(subproject, "run"), "--console=plain", "-PbesomBootstrapJar=" + bootstrapLibJarPath, "-PmainClass=besom.bootstrap.PulumiPluginsDiscoverer", + "--args=--output-file " + pluginDiscovererOutputPath, }, VersionArgs: []string{"--version"}, } diff --git a/language-plugin/pulumi-language-scala/executors/executor_jar.go b/language-plugin/pulumi-language-scala/executors/executor_jar.go index 96b334a3..48feb23d 100644 --- a/language-plugin/pulumi-language-scala/executors/executor_jar.go +++ b/language-plugin/pulumi-language-scala/executors/executor_jar.go @@ -25,10 +25,11 @@ func (j jarexec) NewScalaExecutor(opts ScalaExecutorOptions) (*ScalaExecutor, er if err != nil { return nil, err } - return j.newJarExecutor(cmd, opts.BootstrapLibJarPath, opts.Binary) + pluginDiscovererOutputPath := PluginDiscovererOutputFilePath(opts.WD) + return j.newJarExecutor(cmd, opts.BootstrapLibJarPath, opts.Binary, pluginDiscovererOutputPath) } -func (jarexec) newJarExecutor(cmd string, bootstrapLibJarPath string, rawBinaryPath string) (*ScalaExecutor, error) { +func (jarexec) newJarExecutor(cmd string, bootstrapLibJarPath string, rawBinaryPath string, pluginDiscovererOutputPath string) (*ScalaExecutor, error) { binaryPath := filepath.Clean(rawBinaryPath) classPath := bootstrapLibJarPath + ":" + binaryPath @@ -37,7 +38,7 @@ func (jarexec) newJarExecutor(cmd string, bootstrapLibJarPath string, rawBinaryP Cmd: cmd, BuildArgs: nil, // not supported RunArgs: []string{"-jar", binaryPath}, - PluginArgs: []string{"-cp", classPath, "besom.bootstrap.PulumiPluginsDiscoverer"}, + PluginArgs: []string{"-cp", classPath, "besom.bootstrap.PulumiPluginsDiscoverer", "--output-file", pluginDiscovererOutputPath}, VersionArgs: []string{"-version"}, }, nil } diff --git a/language-plugin/pulumi-language-scala/executors/executor_maven.go b/language-plugin/pulumi-language-scala/executors/executor_maven.go index 577f28da..0e0ccadf 100644 --- a/language-plugin/pulumi-language-scala/executors/executor_maven.go +++ b/language-plugin/pulumi-language-scala/executors/executor_maven.go @@ -29,7 +29,8 @@ func (m maven) NewScalaExecutor(opts ScalaExecutorOptions) (*ScalaExecutor, erro if err != nil { return nil, err } - executor, err := m.newMavenExecutor(cmd, opts.BootstrapLibJarPath) + pluginDiscovererOutputPath := PluginDiscovererOutputFilePath(opts.WD) + executor, err := m.newMavenExecutor(cmd, opts.BootstrapLibJarPath, pluginDiscovererOutputPath) if err != nil { return nil, err } @@ -58,7 +59,7 @@ func (maven) isMavenProject(opts ScalaExecutorOptions) (bool, error) { return fsys.FileExists(opts.WD, "pom.xml") } -func (maven) newMavenExecutor(cmd string, bootstrapLibJarPath string) (*ScalaExecutor, error) { +func (maven) newMavenExecutor(cmd string, bootstrapLibJarPath string, pluginDiscovererOutputPath string) (*ScalaExecutor, error) { return &ScalaExecutor{ Name: "maven", Cmd: cmd, @@ -75,12 +76,11 @@ func (maven) newMavenExecutor(cmd string, bootstrapLibJarPath string) (*ScalaExe "scala:run", }, PluginArgs: []string{ - /* move normal output to STDERR, because we need STDOUT for JSON with plugin results */ - "-Dorg.slf4j.simpleLogger.logFile=System.err", "--no-transfer-progress", "-DbesomBootstrapJar=" + bootstrapLibJarPath, "-DmainClass=besom.bootstrap.PulumiPluginsDiscoverer", "scala:run", + "-DaddArgs=--output-file|" + pluginDiscovererOutputPath, }, VersionArgs: []string{"--version"}, }, nil diff --git a/language-plugin/pulumi-language-scala/executors/executor_sbt.go b/language-plugin/pulumi-language-scala/executors/executor_sbt.go index f09b68f2..1c178866 100644 --- a/language-plugin/pulumi-language-scala/executors/executor_sbt.go +++ b/language-plugin/pulumi-language-scala/executors/executor_sbt.go @@ -30,7 +30,8 @@ func (s sbt) NewScalaExecutor(opts ScalaExecutorOptions) (*ScalaExecutor, error) if err != nil { return nil, err } - return s.newSbtExecutor(cmd, opts.BootstrapLibJarPath) + pluginDiscovererOutputPath := PluginDiscovererOutputFilePath(opts.WD) + return s.newSbtExecutor(cmd, opts.BootstrapLibJarPath, pluginDiscovererOutputPath) } func (sbt) isSbtProject(opts ScalaExecutorOptions) (bool, error) { @@ -53,7 +54,7 @@ func (sbt) isSbtProject(opts ScalaExecutorOptions) (bool, error) { return false, nil } -func (sbt) newSbtExecutor(cmd string, bootstrapLibJarPath string) (*ScalaExecutor, error) { +func (sbt) newSbtExecutor(cmd string, bootstrapLibJarPath string, pluginDiscovererOutputPath string) (*ScalaExecutor, error) { sbtModule := os.Getenv("BESOM_SBT_MODULE") se := &ScalaExecutor{ @@ -61,23 +62,21 @@ func (sbt) newSbtExecutor(cmd string, bootstrapLibJarPath string) (*ScalaExecuto Cmd: cmd, RunArgs: makeArgs(sbtModule, "run"), BuildArgs: makeArgs(sbtModule, "compile"), - PluginArgs: append([]string{"-batch", "-error"}, makePluginsSbtCommandParts(sbtModule, bootstrapLibJarPath)), + PluginArgs: []string{"-batch", makePluginsSbtCommandParts(sbtModule, bootstrapLibJarPath, pluginDiscovererOutputPath)}, VersionArgs: []string{"--numeric-version"}, } return se, nil } -func makePluginsSbtCommandParts(sbtModule string, bootstrapLibJarPath string) string { +func makePluginsSbtCommandParts(sbtModule string, bootstrapLibJarPath string, pluginDiscovererOutputPath string) string { if sbtModule != "" { sbtModule = sbtModule + " / " } pluginsSbtCommandParts := []string{ - // STDOUT needs to be clean of sbt output, because we expect a JSON with plugin results - `; set outputStrategy := Some(StdoutOutput)`, fmt.Sprintf(`; set %sCompile / unmanagedJars += Attributed.blank(file("%s"))`, sbtModule, bootstrapLibJarPath), - fmt.Sprintf(`; %srunMain besom.bootstrap.PulumiPluginsDiscoverer`, sbtModule), + fmt.Sprintf(`; %srunMain besom.bootstrap.PulumiPluginsDiscoverer --output-file %s`, sbtModule, pluginDiscovererOutputPath), } pluginsSbtCommand := strings.Join(pluginsSbtCommandParts, " ") @@ -86,7 +85,7 @@ func makePluginsSbtCommandParts(sbtModule string, bootstrapLibJarPath string) st func makeArgs(sbtModule string, cmd string) []string { if sbtModule != "" { - return append([]string{"-batch", fmt.Sprintf("%s/%s", sbtModule, cmd)}) + return []string{"-batch", fmt.Sprintf("%s/%s", sbtModule, cmd)} } return []string{"-batch", cmd} } diff --git a/language-plugin/pulumi-language-scala/executors/executor_scala_cli.go b/language-plugin/pulumi-language-scala/executors/executor_scala_cli.go index 4a510741..eff01a68 100644 --- a/language-plugin/pulumi-language-scala/executors/executor_scala_cli.go +++ b/language-plugin/pulumi-language-scala/executors/executor_scala_cli.go @@ -21,17 +21,18 @@ func (s scalacli) NewScalaExecutor(opts ScalaExecutorOptions) (*ScalaExecutor, e if err != nil { return nil, err } - return s.newScalaCliExecutor(cmd, opts.BootstrapLibJarPath) + pluginDiscovererOutputPath := PluginDiscovererOutputFilePath(opts.WD) + return s.newScalaCliExecutor(cmd, opts.BootstrapLibJarPath, pluginDiscovererOutputPath) } -func (scalacli) newScalaCliExecutor(cmd string, bootstrapLibJarPath string) (*ScalaExecutor, error) { +func (scalacli) newScalaCliExecutor(cmd string, bootstrapLibJarPath string, pluginDiscovererOutputPath string) (*ScalaExecutor, error) { scalaCliOpts := os.Getenv("BESOM_LANGHOST_SCALA_CLI_OPTS") return &ScalaExecutor{ Name: "scala-cli", Cmd: cmd, BuildArgs: []string{"compile", ".", scalaCliOpts}, RunArgs: []string{"run", ".", scalaCliOpts}, - PluginArgs: []string{"run", ".", scalaCliOpts, "--jar", bootstrapLibJarPath, "--main-class", "besom.bootstrap.PulumiPluginsDiscoverer"}, + PluginArgs: []string{"run", ".", scalaCliOpts, "--jar", bootstrapLibJarPath, "--main-class", "besom.bootstrap.PulumiPluginsDiscoverer", "--", "--output-file", pluginDiscovererOutputPath}, VersionArgs: []string{"version", "--cli", "--offline"}, }, nil } diff --git a/language-plugin/pulumi-language-scala/main.go b/language-plugin/pulumi-language-scala/main.go index 6d9884ec..7b44ccbc 100644 --- a/language-plugin/pulumi-language-scala/main.go +++ b/language-plugin/pulumi-language-scala/main.go @@ -210,10 +210,22 @@ func (host *scalaLanguageHost) determinePulumiPackages( return []plugin.PulumiPluginJSON{}, nil } - logging.V(5).Infof("GetRequiredPlugins: bootstrap raw output=%v", output) + projectRoot := host.execOptions.WD + outputFilePath := executors.PluginDiscovererOutputFilePath(projectRoot) + defer os.Remove(outputFilePath) + + outputFileBytes, err := os.ReadFile(outputFilePath) + + if err != nil { + logging.V(3).Infof("language host failed to read project's plugins from file, "+ + "returning empty plugins; cause: %s", err) + return []plugin.PulumiPluginJSON{}, nil + } + + logging.V(5).Infof("GetRequiredPlugins: bootstrap raw output=%v", string(outputFileBytes)) var plugins []plugin.PulumiPluginJSON - err = json.Unmarshal([]byte(output.stdout), &plugins) + err = json.Unmarshal(outputFileBytes, &plugins) if err != nil { if e, ok := err.(*json.SyntaxError); ok { logging.V(5).Infof("JSON syntax error at byte offset %d", e.Offset) From 6f542acedaac42c3459a1339f1130f8457659fad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pa=C5=82ka?= Date: Wed, 25 Sep 2024 10:39:57 +0200 Subject: [PATCH 2/5] Fix formatting --- language-plugin/bootstrap/PulumiPlugins.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/language-plugin/bootstrap/PulumiPlugins.scala b/language-plugin/bootstrap/PulumiPlugins.scala index a99017fb..a20a7178 100644 --- a/language-plugin/bootstrap/PulumiPlugins.scala +++ b/language-plugin/bootstrap/PulumiPlugins.scala @@ -40,8 +40,9 @@ object PulumiPluginsDiscoverer: JsArray(foundPlugins.values.toVector).compactPrint private def writeToFile(path: String, text: String): Unit = - val file = new File(path) + val file = new File(path) val writer = new PrintWriter(file) writer.write(text) writer.close() +end PulumiPluginsDiscoverer From 43251df2c0b1d107783a998daabb0f8b233dfa7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pa=C5=82ka?= Date: Wed, 25 Sep 2024 14:57:58 +0200 Subject: [PATCH 3/5] Try to speed up builds --- Justfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Justfile b/Justfile index dfb8187b..1e093252 100644 --- a/Justfile +++ b/Justfile @@ -412,7 +412,7 @@ compile-codegen: publish-local-model scala-cli --power compile {{no-bloop}} codegen --suppress-experimental-feature-warning # Runs tests for Besom codegen -test-codegen: compile-codegen +test-codegen: scala-cli --power test {{no-bloop}} codegen --suppress-experimental-feature-warning # Cleans codegen build From 1a04b4678bd295e7a081f21b4a070d43d78411e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pa=C5=82ka?= Date: Wed, 25 Sep 2024 15:56:48 +0200 Subject: [PATCH 4/5] Try to speed up builds --- integration-tests/LanguagePluginTest.test.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/integration-tests/LanguagePluginTest.test.scala b/integration-tests/LanguagePluginTest.test.scala index 6df7b45c..edcd9509 100644 --- a/integration-tests/LanguagePluginTest.test.scala +++ b/integration-tests/LanguagePluginTest.test.scala @@ -199,10 +199,12 @@ class LanguagePluginTest extends munit.FunSuite { clue(actual) == clue(expectedBootstrapPluginsJson) } + val pulumiUpEnv = ctx.env + ("PULUMI_DISABLE_AUTOMATIC_PLUGIN_ACQUISITION" -> "false") + val pulumiUpOutput = pulumi .up(ctx.stackName, "--skip-preview") - .call(cwd = ctx.programDir, env = ctx.env) + .call(cwd = ctx.programDir, env = pulumiUpEnv) .out .text() From 3c63b076cc14ccc0b4d5fd2971856bd157f9480c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pa=C5=82ka?= Date: Mon, 7 Oct 2024 14:38:40 +0200 Subject: [PATCH 5/5] Don't disable automatic plugin aquisition in tests --- integration-tests/LanguagePluginTest.test.scala | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/integration-tests/LanguagePluginTest.test.scala b/integration-tests/LanguagePluginTest.test.scala index edcd9509..6df7b45c 100644 --- a/integration-tests/LanguagePluginTest.test.scala +++ b/integration-tests/LanguagePluginTest.test.scala @@ -199,12 +199,10 @@ class LanguagePluginTest extends munit.FunSuite { clue(actual) == clue(expectedBootstrapPluginsJson) } - val pulumiUpEnv = ctx.env + ("PULUMI_DISABLE_AUTOMATIC_PLUGIN_ACQUISITION" -> "false") - val pulumiUpOutput = pulumi .up(ctx.stackName, "--skip-preview") - .call(cwd = ctx.programDir, env = pulumiUpEnv) + .call(cwd = ctx.programDir, env = ctx.env) .out .text()