Skip to content

Commit

Permalink
Update besom scripts
Browse files Browse the repository at this point in the history
  • Loading branch information
@pawelprazak
pawelprazak committed Feb 10, 2024
1 parent 5a08f0b commit d80a2eb
Show file tree
Hide file tree
Showing 3 changed files with 128 additions and 25 deletions.
120 changes: 98 additions & 22 deletions scripts/Packages.scala
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,16 @@
package besom.scripts

import besom.codegen.*
import besom.codegen.Config.CodegenConfig
import besom.codegen.{PackageMetadata, PackageVersion, UpickleApi, generator}
import besom.model.SemanticVersion
import coursier.error.ResolutionError.CantDownloadModule
import org.virtuslab.yaml.*
import os.Path
import os.{Path, Shellable}

import java.util.Date
import scala.collection.immutable.ListMap
import scala.collection.mutable
import scala.concurrent.duration.*
import scala.sys.exit
import scala.util.*
import scala.util.control.NonFatal
Expand All @@ -16,21 +19,22 @@ import scala.util.control.NonFatal
object Packages:
def main(args: String*): Unit =
args match
case "list" :: Nil => listLatestPackages(packagesDir)
case "local-all" :: Nil => publishLocalAll(generateAll(packagesDir))
case "local" :: tail => publishLocalSelected(generateSelected(packagesDir, tail), tail)
case "maven-all" :: Nil => publishMavenAll(generateAll(packagesDir))
case "maven" :: tail => publishMavenSelected(generateSelected(packagesDir, tail), tail)
case "metadata-all" :: Nil => downloadPackagesMetadata(packagesDir)
case "metadata" :: tail => downloadPackagesMetadata(packagesDir, selected = tail)
case "generate-all" :: Nil => generateAll(packagesDir)
case "generate" :: tail => generateSelected(packagesDir, tail)
case "publish-local-all" :: Nil => publishLocalAll(generatedFile)
case "publish-local" :: tail => publishLocalSelected(generatedFile, tail)
case "publish-maven-all" :: Nil => publishMavenAll(generatedFile)
case "publish-maven" :: tail => publishMavenSelected(generatedFile, tail)
case "compile" :: tail => compileSelected(generatedFile, tail)
case "publish-maven-no-deps" :: tail => publishMavenSelectedNoDeps(generatedFile, tail)
case "list" :: Nil => listLatestPackages(packagesDir)
case "list-published-to-maven" :: Nil => listPublishedPackages(publishedMavenFile)
case "local-all" :: Nil => publishLocalAll(generateAll(packagesDir))
case "local" :: tail => publishLocalSelected(generateSelected(packagesDir, tail), tail)
case "maven-all" :: Nil => publishMavenAll(generateAll(packagesDir))
case "maven" :: tail => publishMavenSelected(generateSelected(packagesDir, tail), tail)
case "metadata-all" :: Nil => downloadPackagesMetadata(packagesDir)
case "metadata" :: tail => downloadPackagesMetadata(packagesDir, selected = tail)
case "generate-all" :: Nil => generateAll(packagesDir)
case "generate" :: tail => generateSelected(packagesDir, tail)
case "publish-local-all" :: Nil => publishLocalAll(generatedFile)
case "publish-local" :: tail => publishLocalSelected(generatedFile, tail)
case "publish-maven-all" :: Nil => publishMavenAll(generatedFile)
case "publish-maven" :: tail => publishMavenSelected(generatedFile, tail)
case "compile" :: tail => compileSelected(generatedFile, tail)
case "publish-maven-no-deps" :: tail => publishMavenSelectedNoDeps(generatedFile, tail)
case cmd =>
println(s"Unknown command: $cmd\n")
println(
Expand Down Expand Up @@ -120,6 +124,7 @@ object Packages:

private val compileProblemPackages = blockedPackages ++ Vector(
"azure-native", // does not compile in finite time
"azure-quickstart-acr-geo-replication", // depends on azure-native
"aws-iam", // id parameter, schema error - components should make this viable
"nuage" // id parameter, schema error - components should make this viable
)
Expand Down Expand Up @@ -240,6 +245,7 @@ object Packages:
Progress.report(label = s"${m.name}:${m.version.getOrElse("latest")}")
val version = m.version.getOrElse(throw Exception("Package version must be provided for publishing")).asString
try
awaitPublishedToMaven(m)
val args: Seq[os.Shellable] = Seq(
"scala-cli",
"--power",
Expand Down Expand Up @@ -385,10 +391,9 @@ object Packages:
val version = m.version.getOrElse(throw Exception("Package version must be provided for publishing")).asString
Progress.report(label = s"${m.name}:${version}")
val logFile = publishMavenDir / s"${m.name}-${version}.log"
println("Waiting for 5 seconds to allow the previous publish to propagate")
Thread.sleep(5000)
try
val args: Seq[os.Shellable] = Seq(
awaitPublishedToMaven(m)
val args: Seq[Shellable] = Seq(
"scala-cli",
"--power",
"publish",
Expand All @@ -402,6 +407,12 @@ object Packages:
done += m
Progress.total(todo.size)
catch
case e: CantDownloadModule =>
e.printStackTrace()
Progress.fail(
s"[${new Date}] Publish failed for provider '${m.name}' version '${version}' " +
s"error: failed to download dependency, message: ${e.getMessage}"
)
case _: os.SubprocessException =>
Progress.fail(
s"[${new Date}] Publish failed for provider '${m.name}' version '${version}', logs: ${logFile}, " +
Expand All @@ -413,10 +424,54 @@ object Packages:
s"error [${e.getClass.getSimpleName}]: ${e.getMessage}"
)
finally Progress.end
end try
end if
}
done.toVector
}

private def awaitPublishedToMaven(m: PackageMetadata): Unit = {
// Check if dependencies are available in Maven Central before publishing
if m.dependencies.nonEmpty then
val timeout = 10.minutes
val interval = 10.second
val deadline = timeout.fromNow
println(s"Checking for dependencies with timeout of ${timeout}")

// TODO: clean this up, it's a bit of a mess
val defaultScalaVersion = Config.DefaultBesomVersion
val shortCoreVersion = SemanticVersion
.parseTolerant(defaultScalaVersion)
.fold(
e => Left(Exception(s"Invalid besom version: ${defaultScalaVersion}", e)),
v => Right(v.copy(patch = 0).toShortString /* drop patch version */)
)

def dependency(m: PackageMetadata): String =
s"org.virtuslab::besom-${m.name}:${m.version.get}-core.${shortCoreVersion.toTry.get}"

val allDependencies = m.dependencies.map(dependency)
val found = mutable.Map.empty[String, Boolean]

def foundAll = found.values.forall(identity)

allDependencies.foreach { d =>
found(d) = false
}
while (deadline.hasTimeLeft && !foundAll) {
allDependencies.foreach { d =>
resolveMavenPackage(d, defaultScalaVersion).fold(
_ => println(s"Dependency '${d}' not found"),
_ => found(d) = true
)
}
if !foundAll then
println(s"Waiting for ${interval} to allow the publish to propagate")
Thread.sleep(interval.toMillis)
}
end if
}

def readPackagesMetadata(targetPath: os.Path, selected: List[String] = Nil): Vector[PackageMetadata] = {
// download metadata if none found
if !os.exists(targetPath) then downloadPackagesMetadata(targetPath, selected)
Expand Down Expand Up @@ -601,10 +656,31 @@ object Packages:

def listLatestPackages(targetPath: os.Path): Unit = {
val metadata = readPackagesMetadata(targetPath)
println(s"Found ${metadata.size} packages in Pulumi repository")
metadata.foreach(m => println(s"${m.name}:${m.version.getOrElse("latest")}"))
val active = metadata
.filterNot(m => blockedPackages.contains(m.name))
.filterNot(m => codegenProblemPackages.contains(m.name))
.filterNot(m => compileProblemPackages.contains(m.name))
.size
println(s"Found ${metadata.size} (${active} active) packages in Pulumi repository")
metadata.foreach(m =>
println {
s"${m.name}:${m.version.getOrElse("latest")}" + {
if blockedPackages.contains(m.name) then " (blocked)" else ""
} + {
if codegenProblemPackages.contains(m.name) then " (codegen problem)" else ""
} + {
if compileProblemPackages.contains(m.name) then " (compile problem)" else ""
}
}
)
}

def listPublishedPackages(file: os.Path): Unit =
val metadata = PackageMetadata.fromJsonList(os.read(file))
println(s"Found ${metadata.size} packages in logs published to Maven")
metadata.foreach(m => println(s"${m.name}:${m.version.getOrElse("unknown")}"))
end listPublishedPackages

def upsertGeneratedFile(metadata: Vector[PackageMetadata]): os.Path = {
val generated =
if os.exists(generatedFile)
Expand Down
26 changes: 26 additions & 0 deletions scripts/common.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -227,3 +227,29 @@ def withProgress[A](title: String, initialTotal: Int)(f: Progress ?=> A): A =
println()

end withProgress

def resolveMavenPackage(name: String, defaultScalaVersion: String): Either[Exception, String] =
import coursier.*
import coursier.parse.DependencyParser
import scala.concurrent.ExecutionContext.Implicits.global

DependencyParser
.dependency(name, defaultScalaVersion)
.fold(
msg => Left(Exception(s"Failed to parse $name: $msg")),
Right(_)
)
.flatMap { dep =>
Resolve()
.addRepositories(
Repositories.central,
Repositories.sonatype("public"),
Repositories.sonatype("snapshots")
)
.addDependencies(dep)
.either()
.flatMap {
_.reconciledVersions.get(dep.module).toRight(Exception(s"Failed to resolve $name"))
}
}
end resolveMavenPackage
7 changes: 4 additions & 3 deletions scripts/project.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,9 @@
//> using dep com.lihaoyi::requests:0.8.0
//> using dep com.lihaoyi::upickle:3.1.4
//> using dep org.apache.commons:commons-lang3:3.14.0
//> using dep "org.scoverage::scalac-scoverage-reporter:2.0.11"
//> using dep "org.scoverage::scalac-scoverage-domain:2.0.11"
//> using dep "org.scoverage::scalac-scoverage-serializer:2.0.11"
//> using dep org.scoverage::scalac-scoverage-reporter:2.0.11
//> using dep org.scoverage::scalac-scoverage-domain:2.0.11
//> using dep org.scoverage::scalac-scoverage-serializer:2.0.11
//> using dep org.virtuslab::scala-yaml:0.0.8
//> using dep io.get-coursier:coursier_2.13:2.1.9
//> using dep org.virtuslab::besom-codegen:0.2.0

0 comments on commit d80a2eb

Please sign in to comment.