Implement crate publishing on CI (#12768)

joao-paulo-parity · rcny · web-flow · commit b99f2bc9c912 · 2022-12-07T18:08:48.000Z
* implement crate publishing from CI * fix indentation * use resource_group for job exclusivity ensure that at most one instance of the publish-crates job is running at any given time to prevent race conditions * correct publish = false * Remove YAML anchors as GitLab's `extends:` doesn't need it * Temporarily force cache upload for the new jobs * Revert `RUSTY_CACHIER_FORCE_UPLOAD` * pin libp2p-tcp=0.37.0 for sc-telemetry * Revert "pin libp2p-tcp=0.37.0 for sc-telemetry" This reverts commit 29146bf. * always collect generated crates * increase timeout for publish-crates-template * Force upload the new job cache again * Revert "Force upload the new job cache again" This reverts commit 5a5feee. * reformat * improve timeout explanation * s/usual/average Co-authored-by: Vladimir Istyufeev <vladimir@parity.io>
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -180,6 +180,25 @@ default:
     # this job runs only on nightly pipeline with the mentioned variable, against `master` branch
     - if: $CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "schedule" && $PIPELINE == "nightly"
 
+.crates-publishing-template:
+  stage:                           test
+  extends:                         .docker-env
+  # collect artifacts even on failure so that we know how the crates were generated (they'll be
+  # generated to the artifacts folder according to SPUB_TMP further down)
+  artifacts:
+    name:                          "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}"
+    when:                          always
+    expire_in:                     7 days
+    paths:
+      - artifacts/
+  variables:
+    CRATESIO_API: https://crates.io/api
+    CRATESIO_CRATES_OWNER: parity-crate-owner
+    GH_API: https://api.github.com
+    REPO: substrate
+    REPO_OWNER: paritytech
+    SPUB_TMP: artifacts
+
 #### stage:                       .pre
 
 skip-if-draft:
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -8,6 +8,7 @@ members = [
 	"bin/node/bench",
 	"bin/node/cli",
 	"bin/node/executor",
+	"bin/node/inspect",
 	"bin/node/primitives",
 	"bin/node/rpc",
 	"bin/node/runtime",
@@ -39,11 +40,13 @@ members = [
 	"client/executor/wasmi",
 	"client/executor/wasmtime",
 	"client/finality-grandpa",
+	"client/finality-grandpa/rpc",
 	"client/informant",
 	"client/keystore",
 	"client/merkle-mountain-range",
 	"client/merkle-mountain-range/rpc",
 	"client/network",
+	"client/network/transactions",
 	"client/network-gossip",
 	"client/network/bitswap",
 	"client/network/common",
@@ -88,11 +91,13 @@ members = [
 	"frame/child-bounties",
 	"frame/collective",
 	"frame/contracts",
+	"frame/contracts/proc-macro",
 	"frame/contracts/primitives",
 	"frame/conviction-voting",
 	"frame/democracy",
 	"frame/fast-unstake",
 	"frame/try-runtime",
+	"frame/elections-phragmen",
 	"frame/election-provider-multi-phase",
 	"frame/election-provider-support",
 	"frame/election-provider-support/benchmarking",
@@ -113,6 +118,7 @@ members = [
 	"frame/nicks",
 	"frame/node-authorization",
 	"frame/offences",
+	"frame/offences/benchmarking",
 	"frame/preimage",
 	"frame/proxy",
 	"frame/nomination-pools",
@@ -143,6 +149,7 @@ members = [
 	"frame/support/procedural/tools/derive",
 	"frame/support/test",
 	"frame/support/test/compile_pass",
+	"frame/support/test/pallet",
 	"frame/system",
 	"frame/system/benchmarking",
 	"frame/system/rpc/runtime-api",
@@ -174,6 +181,7 @@ members = [
 	"primitives/consensus/babe",
 	"primitives/consensus/common",
 	"primitives/consensus/pow",
+	"primitives/consensus/slots",
 	"primitives/consensus/vrf",
 	"primitives/core",
 	"primitives/core/hashing",
@@ -215,6 +223,7 @@ members = [
 	"primitives/version/proc-macro",
 	"primitives/wasm-interface",
 	"primitives/weights",
+	"test-utils",
 	"test-utils/client",
 	"test-utils/derive",
 	"test-utils/runtime",
diff --git a/bin/node/bench/Cargo.toml b/bin/node/bench/Cargo.toml
@@ -7,6 +7,7 @@ edition = "2021"
 license = "GPL-3.0-or-later WITH Classpath-exception-2.0"
 homepage = "https://substrate.io"
 repository = "https://github.com/paritytech/substrate/"
+publish = false
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
diff --git a/bin/node/cli/Cargo.toml b/bin/node/cli/Cargo.toml
@@ -9,6 +9,7 @@ license = "GPL-3.0-or-later WITH Classpath-exception-2.0"
 default-run = "substrate"
 homepage = "https://substrate.io"
 repository = "https://github.com/paritytech/substrate/"
+publish = false
 
 [package.metadata.wasm-pack.profile.release]
 # `wasm-opt` has some problems on linux, see
diff --git a/bin/node/executor/Cargo.toml b/bin/node/executor/Cargo.toml
@@ -7,6 +7,7 @@ edition = "2021"
 license = "Apache-2.0"
 homepage = "https://substrate.io"
 repository = "https://github.com/paritytech/substrate/"
+publish = false
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]
diff --git a/bin/node/inspect/Cargo.toml b/bin/node/inspect/Cargo.toml
@@ -7,6 +7,7 @@ edition = "2021"
 license = "GPL-3.0-or-later WITH Classpath-exception-2.0"
 homepage = "https://substrate.io"
 repository = "https://github.com/paritytech/substrate/"
+publish = false
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]
diff --git a/bin/node/primitives/Cargo.toml b/bin/node/primitives/Cargo.toml
@@ -7,6 +7,7 @@ edition = "2021"
 license = "Apache-2.0"
 homepage = "https://substrate.io"
 repository = "https://github.com/paritytech/substrate/"
+publish = false
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]
diff --git a/bin/node/rpc/Cargo.toml b/bin/node/rpc/Cargo.toml
@@ -7,6 +7,7 @@ edition = "2021"
 license = "Apache-2.0"
 homepage = "https://substrate.io"
 repository = "https://github.com/paritytech/substrate/"
+publish = false
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]
diff --git a/bin/node/runtime/Cargo.toml b/bin/node/runtime/Cargo.toml
@@ -8,6 +8,7 @@ build = "build.rs"
 license = "Apache-2.0"
 homepage = "https://substrate.io"
 repository = "https://github.com/paritytech/substrate/"
+publish = false
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]
diff --git a/bin/node/testing/Cargo.toml b/bin/node/testing/Cargo.toml
@@ -7,7 +7,7 @@ edition = "2021"
 license = "GPL-3.0-or-later WITH Classpath-exception-2.0"
 homepage = "https://substrate.io"
 repository = "https://github.com/paritytech/substrate/"
-publish = true
+publish = false
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]
diff --git a/client/merkle-mountain-range/rpc/Cargo.toml b/client/merkle-mountain-range/rpc/Cargo.toml
@@ -7,7 +7,6 @@ license = "Apache-2.0"
 homepage = "https://substrate.io"
 repository = "https://github.com/paritytech/substrate/"
 description = "Node-specific RPC methods for interaction with Merkle Mountain Range pallet."
-publish = false
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]
diff --git a/client/network/bitswap/Cargo.toml b/client/network/bitswap/Cargo.toml
@@ -8,7 +8,6 @@ edition = "2021"
 homepage = "https://substrate.io"
 repository = "https://github.com/paritytech/substrate/"
 documentation = "https://docs.rs/sc-network-bitswap"
-readme = "README.md"
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]
diff --git a/client/network/common/Cargo.toml b/client/network/common/Cargo.toml
@@ -8,7 +8,6 @@ edition = "2021"
 homepage = "https://substrate.io"
 repository = "https://github.com/paritytech/substrate/"
 documentation = "https://docs.rs/sc-network-sync"
-readme = "README.md"
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]
diff --git a/client/network/light/Cargo.toml b/client/network/light/Cargo.toml
@@ -8,7 +8,6 @@ edition = "2021"
 homepage = "https://substrate.io"
 repository = "https://github.com/paritytech/substrate/"
 documentation = "https://docs.rs/sc-network-light"
-readme = "README.md"
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]
diff --git a/client/network/sync/Cargo.toml b/client/network/sync/Cargo.toml
@@ -8,7 +8,6 @@ edition = "2021"
 homepage = "https://substrate.io"
 repository = "https://github.com/paritytech/substrate/"
 documentation = "https://docs.rs/sc-network-sync"
-readme = "README.md"
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]
diff --git a/client/network/transactions/Cargo.toml b/client/network/transactions/Cargo.toml
@@ -8,7 +8,6 @@ edition = "2021"
 homepage = "https://substrate.io"
 repository = "https://github.com/paritytech/substrate/"
 documentation = "https://docs.rs/sc-network-transactions"
-readme = "README.md"
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]
diff --git a/client/sync-state-rpc/Cargo.toml b/client/sync-state-rpc/Cargo.toml
@@ -7,7 +7,6 @@ edition = "2021"
 license = "Apache-2.0"
 homepage = "https://substrate.io"
 repository = "https://github.com/paritytech/substrate/"
-readme = "README.md"
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]
diff --git a/frame/bags-list/Cargo.toml b/frame/bags-list/Cargo.toml
@@ -7,7 +7,6 @@ license = "Apache-2.0"
 homepage = "https://substrate.io"
 repository = "https://github.com/paritytech/substrate/"
 description = "FRAME pallet bags list"
-readme = "README.md"
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]
diff --git a/frame/bags-list/fuzzer/Cargo.toml b/frame/bags-list/fuzzer/Cargo.toml
@@ -7,7 +7,6 @@ license = "Apache-2.0"
 homepage = "https://substrate.io"
 repository = "https://github.com/paritytech/substrate/"
 description = "Fuzzer for FRAME pallet bags list"
-readme = "README.md"
 publish = false
 
 [dependencies]
diff --git a/frame/bags-list/remote-tests/Cargo.toml b/frame/bags-list/remote-tests/Cargo.toml
@@ -7,7 +7,7 @@ license = "Apache-2.0"
 homepage = "https://substrate.io"
 repository = "https://github.com/paritytech/substrate/"
 description = "FRAME pallet bags list remote test"
-readme = "README.md"
+publish = false
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]
diff --git a/frame/election-provider-multi-phase/Cargo.toml b/frame/election-provider-multi-phase/Cargo.toml
@@ -7,7 +7,6 @@ license = "Apache-2.0"
 homepage = "https://substrate.io"
 repository = "https://github.com/paritytech/substrate/"
 description = "PALLET two phase election providers"
-readme = "README.md"
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]
diff --git a/frame/election-provider-support/Cargo.toml b/frame/election-provider-support/Cargo.toml
@@ -7,7 +7,6 @@ license = "Apache-2.0"
 homepage = "https://substrate.io"
 repository = "https://github.com/paritytech/substrate/"
 description = "election provider supporting traits"
-readme = "README.md"
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]
diff --git a/frame/fast-unstake/Cargo.toml b/frame/fast-unstake/Cargo.toml
@@ -7,7 +7,6 @@ license = "Unlicense"
 homepage = "https://substrate.io"
 repository = "https://github.com/paritytech/substrate/"
 description = "FRAME fast unstake pallet"
-readme = "README.md"
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]
@@ -26,7 +25,10 @@ sp-std = { version = "5.0.0", default-features = false, path = "../../primitives
 sp-staking = { default-features = false, path = "../../primitives/staking" }
 frame-election-provider-support = { default-features = false, path = "../election-provider-support" }
 
+# optional dependencies for cargo features
 frame-benchmarking = { version = "4.0.0-dev", default-features = false, optional = true, path = "../benchmarking" }
+pallet-staking = { default-features = false, optional = true, path = "../staking" }
+pallet-assets = { default-features = false, optional = true, path = "../assets" }
 
 [dev-dependencies]
 pallet-staking-reward-curve = { version = "4.0.0-dev", path = "../staking/reward-curve" }
@@ -36,6 +38,7 @@ sp-tracing = { version = "6.0.0", path = "../../primitives/tracing" }
 pallet-staking = { path = "../staking" }
 pallet-balances = { path = "../balances" }
 pallet-timestamp = { path = "../timestamp" }
+pallet-assets = { path = "../assets" }
 
 
 [features]
diff --git a/frame/lottery/Cargo.toml b/frame/lottery/Cargo.toml
@@ -7,7 +7,6 @@ license = "Apache-2.0"
 homepage = "https://substrate.io"
 repository = "https://github.com/paritytech/substrate/"
 description = "FRAME Participation Lottery Pallet"
-readme = "README.md"
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]
diff --git a/frame/nomination-pools/benchmarking/Cargo.toml b/frame/nomination-pools/benchmarking/Cargo.toml
@@ -31,6 +31,7 @@ sp-runtime = { version = "7.0.0", default-features = false, path = "../../../pri
 sp-runtime-interface = { version = "7.0.0", default-features = false, path = "../../../primitives/runtime-interface" }
 sp-staking = { version = "4.0.0-dev", default-features = false, path = "../../../primitives/staking" }
 sp-std = { version = "5.0.0", default-features = false, path = "../../../primitives/std" }
+sp-io = { optional = true, default-features = false, path = "../../../primitives/io" }
 
 [dev-dependencies]
 pallet-balances = { version = "4.0.0-dev", default-features = false, path = "../../balances" }
@@ -54,7 +55,7 @@ std = [
 	"sp-runtime-interface/std",
 	"sp-io/std",
 	"sp-staking/std",
-	"sp-std/std",	
+	"sp-std/std",
 ]
 
 runtime-benchmarks = [
diff --git a/frame/nomination-pools/test-staking/Cargo.toml b/frame/nomination-pools/test-staking/Cargo.toml
@@ -7,6 +7,7 @@ license = "Apache-2.0"
 homepage = "https://substrate.io"
 repository = "https://github.com/paritytech/substrate/"
 description = "FRAME nomination pools pallet tests with the staking pallet"
+publish = false
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]
diff --git a/frame/preimage/Cargo.toml b/frame/preimage/Cargo.toml
@@ -7,7 +7,6 @@ license = "Apache-2.0"
 homepage = "https://substrate.io"
 repository = "https://github.com/paritytech/substrate/"
 description = "FRAME pallet for storing preimages of hashes"
-readme = "README.md"
 
 [dependencies]
 codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] }
diff --git a/frame/root-testing/Cargo.toml b/frame/root-testing/Cargo.toml
@@ -8,6 +8,7 @@ homepage = "https://substrate.io"
 repository = "https://github.com/paritytech/substrate/"
 description = "FRAME root testing pallet"
 readme = "README.md"
+publish = false
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]
diff --git a/frame/staking/Cargo.toml b/frame/staking/Cargo.toml
@@ -32,9 +32,10 @@ sp-application-crypto = { version = "7.0.0", default-features = false, path = ".
 frame-election-provider-support = { version = "4.0.0-dev", default-features = false, path = "../election-provider-support" }
 log = { version = "0.4.17", default-features = false }
 
-# Optional imports for benchmarking
+# optional dependencies for cargo features
 frame-benchmarking = { version = "4.0.0-dev", default-features = false, path = "../benchmarking", optional = true }
 rand_chacha = { version = "0.2", default-features = false, optional = true }
+pallet-bags-list = { default-features = false, optional = true, path = "../bags-list" }
 
 [dev-dependencies]
 sp-tracing = { version = "6.0.0", path = "../../primitives/tracing" }
diff --git a/frame/state-trie-migration/Cargo.toml b/frame/state-trie-migration/Cargo.toml
@@ -7,7 +7,6 @@ license = "Apache-2.0"
 homepage = "https://substrate.io"
 repository = "https://github.com/paritytech/substrate/"
 description = "FRAME pallet migration of trie"
-readme = "README.md"
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]
diff --git a/frame/transaction-payment/asset-tx-payment/Cargo.toml b/frame/transaction-payment/asset-tx-payment/Cargo.toml
@@ -19,10 +19,12 @@ sp-io = { version = "7.0.0", default-features = false, path = "../../../primitiv
 sp-runtime = { version = "7.0.0", default-features = false, path = "../../../primitives/runtime" }
 sp-std = { version = "5.0.0", default-features = false, path = "../../../primitives/std" }
 
+# optional dependencies for cargo features
 frame-support = { version = "4.0.0-dev", default-features = false, path = "../../support" }
 frame-system = { version = "4.0.0-dev", default-features = false, path = "../../system" }
 pallet-transaction-payment = { version = "4.0.0-dev", default-features = false, path = ".." }
 frame-benchmarking = { version = "4.0.0-dev", default-features = false, path = "../../benchmarking", optional = true }
+pallet-assets = { default-features = false, optional = true, path = "../../assets" }
 
 # Other dependencies
 codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive"] }
diff --git a/frame/try-runtime/Cargo.toml b/frame/try-runtime/Cargo.toml
@@ -7,7 +7,6 @@ license = "Apache-2.0"
 homepage = "https://substrate.io"
 repository = "https://github.com/paritytech/substrate/"
 description = "FRAME pallet for democracy"
-readme = "README.md"
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]
diff --git a/frame/whitelist/Cargo.toml b/frame/whitelist/Cargo.toml
@@ -7,7 +7,6 @@ license = "Apache-2.0"
 homepage = "https://substrate.io"
 repository = "https://github.com/paritytech/substrate/"
 description = "FRAME pallet for whitelisting call, and dispatch from specific origin"
-readme = "README.md"
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]
diff --git a/primitives/beefy/Cargo.toml b/primitives/beefy/Cargo.toml
@@ -7,7 +7,6 @@ license = "Apache-2.0"
 homepage = "https://substrate.io"
 repository = "https://github.com/paritytech/substrate"
 description = "Primitives for BEEFY protocol."
-readme = "README.md"
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]
diff --git a/primitives/weights/Cargo.toml b/primitives/weights/Cargo.toml
diff --git a/scripts/ci/gitlab/pipeline/publish.yml b/scripts/ci/gitlab/pipeline/publish.yml
diff --git a/scripts/ci/gitlab/pipeline/test.yml b/scripts/ci/gitlab/pipeline/test.yml
diff --git a/scripts/ci/node-template-release/Cargo.toml b/scripts/ci/node-template-release/Cargo.toml
diff --git a/test-utils/Cargo.toml b/test-utils/Cargo.toml
diff --git a/test-utils/client/Cargo.toml b/test-utils/client/Cargo.toml
diff --git a/test-utils/derive/Cargo.toml b/test-utils/derive/Cargo.toml
diff --git a/utils/frame/generate-bags/Cargo.toml b/utils/frame/generate-bags/Cargo.toml
diff --git a/utils/frame/generate-bags/node-runtime/Cargo.toml b/utils/frame/generate-bags/node-runtime/Cargo.toml
diff --git a/utils/frame/remote-externalities/Cargo.toml b/utils/frame/remote-externalities/Cargo.toml
diff --git a/utils/frame/rpc/client/Cargo.toml b/utils/frame/rpc/client/Cargo.toml
diff --git a/utils/frame/try-runtime/cli/Cargo.toml b/utils/frame/try-runtime/cli/Cargo.toml
diff --git a/utils/wasm-builder/src/wasm_project.rs b/utils/wasm-builder/src/wasm_project.rs
