-
Notifications
You must be signed in to change notification settings - Fork 201
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Avoid unnecessary Cargo crate rebuilds
This commit modifies the Gradle buildscripts to avoid unnecessary Cargo rebuilds of the generated crates, decreasing development iteration cycles. Prior to this commit, if you ran a crate-generating command _twice_ on any of the `codegen-test`, `codegen-server-test`, `sdk-codegen-test`, and `codegen-server-test:python` subprojects (without making any changes to the codegen code), both invocations would take the same time: Cargo would recompile the crate and its dependencies, even when the generated crate is identical. This is because Gradle deletes everything under `buildDir` when running the `generateSmithyBuild` task: ``` > Task :codegen-server-test:smithyBuildJar Deleting stale output file: /local/home/davidpz/workplace/smithy-ws/src/SmithyRsSource/codegen-server-test/build/smithyprojections/codegen-server-test ``` So the files get recreated each time. While developing, it is likely that only a small number of the generated crate files are modified across rebuilds. [Cargo uses `mtime`](rust-lang/cargo#6529) (among other factors) to determine whether it needs to recompile a unit. Indeed, running with `CARGO_LOG=cargo::core::compiler::fingerprint=trace` yields `err: current filesystem status shows we're outdated`. This commit adds a Gradle task that compares the hashes of the newly generated files with the (previously cached) old ones, and restores their `mtime`s if the hashes coincide. Another issue was causing unnecessary crate rebuilds. Prior to this commit, we were sending `RUSTFLAGS=-D warnings` when invoking Cargo. However, a common thing to do after generating a crate is to open its contents in an editor. The editor's `rust-analyzer` would compile the crate and its dependencies without the `RUSTFLAGS` we had used earlier. The next time you rebuilt the crate, Cargo would claim `err: RUSTFLAGS has changed: previously [], now ["-D", "warnings"]` and recompile everything again. This commit refactors the Gradle tasks so as to not send these flags when invoking Cargo, instead generating a `.cargo/config.toml` containing these flags. This way, `rust-analyzer` also picks them up and does not need to recompile the crates. With both patches, Cargo avoids unnecessary crate rebuilds. All in all, the second invocation of a `./gradlew --info -P modules='simple' -P cargoCommands='test' codegen-server-test:build` command now takes 15 seconds less than the first invocation on my `c5.4xlarge` machine; Cargo does not need to do _any_ work on the second invocation. This commit also refactors the `build.gradle.kts` files of the `sdk`, `sdk-codegen-test`, `codegen-test`, `codegen-server-test`, and `codegen-server-test:python` subprojects to make them DRYer and more consistent. The last 4 subprojects' buildscripts are now much shorter, with all the common logic having been moved to `CodegenTestCommon.kt`. Note that we have made the last 4 subprojects' `cargo check` and `cargo doc` invocations use the same set of flags than in the `sdk` subproject for consistency. Closes #1378. Closes #1412.
- Loading branch information
1 parent
c503deb
commit 668fbb4
Showing
8 changed files
with
257 additions
and
273 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -8,13 +8,14 @@ extra["moduleName"] = "software.amazon.smithy.kotlin.codegen.test" | |
|
|
||
| tasks["jar"].enabled = false | ||
|
|
||
| plugins { | ||
| id("software.amazon.smithy").version("0.5.3") | ||
| } | ||
| plugins { id("software.amazon.smithy").version("0.5.3") } | ||
|
|
||
| val smithyVersion: String by project | ||
| val defaultRustFlags: String by project | ||
| val defaultRustDocFlags: String by project | ||
| val properties = PropertyRetriever(rootProject, project) | ||
|
|
||
| val pluginName = "rust-codegen" | ||
| val workingDirUnderBuildDir = "smithyprojections/sdk-codegen-test/" | ||
|
|
||
| dependencies { | ||
| implementation(project(":aws:sdk-codegen")) | ||
|
|
@@ -23,94 +24,19 @@ dependencies { | |
| implementation("software.amazon.smithy:smithy-aws-traits:$smithyVersion") | ||
| } | ||
|
|
||
| data class CodegenTest(val service: String, val module: String, val extraConfig: String? = null) | ||
|
|
||
| val CodegenTests = listOf( | ||
| val allCodegenTests = listOf( | ||
| CodegenTest("com.amazonaws.apigateway#BackplaneControlService", "apigateway") | ||
| ) | ||
|
|
||
| fun generateSmithyBuild(tests: List<CodegenTest>): String { | ||
| val projections = tests.joinToString(",\n") { | ||
| """ | ||
| "${it.module}": { | ||
| "plugins": { | ||
| "rust-codegen": { | ||
| "runtimeConfig": { | ||
| "relativePath": "${rootProject.projectDir.absolutePath}/rust-runtime" | ||
| }, | ||
| "service": "${it.service}", | ||
| "module": "${it.module}", | ||
| "moduleAuthors": ["[email protected]"], | ||
| "moduleVersion": "0.0.1" | ||
| ${it.extraConfig ?: ""} | ||
| } | ||
| } | ||
| } | ||
| """.trimIndent() | ||
| } | ||
| return """ | ||
| { | ||
| "version": "1.0", | ||
| "projections": { $projections } | ||
| } | ||
| """ | ||
| } | ||
|
|
||
| task("generateSmithyBuild") { | ||
| description = "generate smithy-build.json" | ||
| doFirst { | ||
| buildDir.resolve("smithy-build.json").writeText(generateSmithyBuild(CodegenTests)) | ||
| } | ||
| } | ||
|
|
||
| fun generateCargoWorkspace(tests: List<CodegenTest>): String { | ||
| return """ | ||
| [workspace] | ||
| members = [ | ||
| ${tests.joinToString(",") { "\"${it.module}/rust-codegen\"" }} | ||
| ] | ||
| """.trimIndent() | ||
| } | ||
| task("generateCargoWorkspace") { | ||
| description = "generate Cargo.toml workspace file" | ||
| doFirst { | ||
| buildDir.resolve("smithyprojections/sdk-codegen-test/Cargo.toml").writeText(generateCargoWorkspace(CodegenTests)) | ||
| } | ||
| } | ||
| registerGenerateSmithyBuildTask(rootProject, project, pluginName, allCodegenTests) | ||
| registerGenerateCargoWorkspaceTask(rootProject, project, pluginName, allCodegenTests, workingDirUnderBuildDir) | ||
|
|
||
| tasks["smithyBuildJar"].dependsOn("generateSmithyBuild") | ||
| tasks["assemble"].finalizedBy("generateCargoWorkspace") | ||
|
|
||
| tasks.register<Exec>("cargoCheck") { | ||
| workingDir("build/smithyprojections/sdk-codegen-test/") | ||
| environment("RUSTFLAGS", defaultRustFlags) | ||
| commandLine("cargo", "check") | ||
| dependsOn("assemble") | ||
| } | ||
|
|
||
| tasks.register<Exec>("cargoTest") { | ||
| workingDir("build/smithyprojections/sdk-codegen-test/") | ||
| environment("RUSTFLAGS", defaultRustFlags) | ||
| commandLine("cargo", "test") | ||
| dependsOn("assemble") | ||
| } | ||
|
|
||
| tasks.register<Exec>("cargoDocs") { | ||
| workingDir("build/smithyprojections/sdk-codegen-test/") | ||
| environment("RUSTDOCFLAGS", defaultRustDocFlags) | ||
| commandLine("cargo", "doc", "--no-deps") | ||
| dependsOn("assemble") | ||
| } | ||
|
|
||
| tasks.register<Exec>("cargoClippy") { | ||
| workingDir("build/smithyprojections/sdk-codegen-test/") | ||
| environment("RUSTFLAGS", defaultRustFlags) | ||
| commandLine("cargo", "clippy") | ||
| dependsOn("assemble") | ||
| } | ||
| registerModifyMtimeTask(project) | ||
| registerCargoCommandsTasks(project, buildDir.resolve(workingDirUnderBuildDir), defaultRustDocFlags) | ||
|
|
||
| tasks["test"].finalizedBy("cargoCheck", "cargoClippy", "cargoTest", "cargoDocs") | ||
| tasks["test"].finalizedBy(cargoCommands(properties).map { it.toString }) | ||
|
|
||
| tasks["clean"].doFirst { | ||
| delete("smithy-build.json") | ||
| } | ||
| tasks["clean"].doFirst { delete("smithy-build.json") } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.