# Goal

This PR aims to remove the `rustup install toolchain nightly` commands from the individual
GitHub jobs that use the `ci-base-image` that previously did not have the `nightly` toolchain.

Closes #1727

 # Discussion
The changes in the previous PR (#1700) to add the `nightly` toolchain to `ci-base-image` were
not published to DockerHub until the `merge-pr.yml` workflow actions were successfully run.
Therefore the `nightly` toolchain was not available for CI to use in `ci-base-image`.

In that PR, the `nightly` toolchain was installed in the individual jobs where it was needed to pass CI.

Now that the PR has been merged, `ci-base-image` has been updated with both the `stable`
and `nightly` toolchains and the redundant `rustup install` commands can be removed from `verify-pr-commit.yml`.

Author: Matthew Orris

.github/workflows/release.yml

@@ -348,7 +348,7 @@ jobs:
         uses: actions/configure-pages@v3
       - name: Build Docs
         run: |
-          rustup target add wasm32-unknown-unknown
+          rustup target add wasm32-unknown-unknown --toolchain nightly-2023-07-13
           RUSTDOCFLAGS="--enable-index-page -Zunstable-options" cargo +nightly-2023-07-13 doc --no-deps --workspace --features frequency
       - name: Fix file permissions
         shell: sh

.github/workflows/rococo.yml

@@ -16,7 +16,7 @@ env:
 jobs:
   run-e2e:
     name: Run E2E Tests
-    runs-on: ubuntu-20.04
+    runs-on: [self-hosted, Linux, X64, build, v2]
     container: ghcr.io/libertydsnp/frequency/ci-base-image
     steps:
       - name: Validate

.github/workflows/verify-pr-commit.yml

@@ -199,10 +199,7 @@ jobs:
       - name: Check Out Repo
         uses: actions/checkout@v4
       - name: Check
-        # Remove these install commands when ci-base-image has nightly toolchain
         run: |
-          rustup toolchain install nightly-2023-07-13
-          rustup target add x86_64-unknown-linux-gnu --toolchain nightly-2023-07-13
           cargo +nightly-2023-07-13 fmt --check
 
   lint-rust-code:
@@ -215,10 +212,7 @@ jobs:
       - name: Check Out Repo
         uses: actions/checkout@v4
       - name: Lint
-        # Remove these install commands when ci-base-image has nightly toolchain
         run: |
-          rustup toolchain install nightly-2023-07-13
-          rustup target add x86_64-unknown-linux-gnu --toolchain nightly-2023-07-13
           SKIP_WASM_BUILD=1 env -u RUSTFLAGS cargo +nightly-2023-07-13 clippy \
             --features runtime-benchmarks,frequency-lint-check \
             -- \
@@ -234,9 +228,7 @@ jobs:
       - name: Check Out Repo
         uses: actions/checkout@v4
       - name: Build Docs
-        # Remove these install commands when ci-base-image has nightly toolchain
         run: |
-          rustup toolchain install nightly-2023-07-13
           rustup target add wasm32-unknown-unknown --toolchain nightly-2023-07-13
           RUSTDOCFLAGS="--enable-index-page --check -Zunstable-options" cargo +nightly-2023-07-13 doc --no-deps --features frequency
 
@@ -643,8 +635,7 @@ jobs:
     if: needs.changes.outputs.run-e2e == 'true'
     needs: [build-binaries, verify-js-api-augment]
     name: Run E2E Tests
-    runs-on: ubuntu-20.04
-    container: ghcr.io/libertydsnp/frequency/ci-base-image
+    runs-on: [self-hosted, Linux, X64, build, v2]
     steps:
       - name: Check Out Repo
         uses: actions/checkout@v4

e2e/package-lock.json

@@ -24,11 +24,13 @@
         "@polkadot/util": "12.5.1",
         "helia": "^2.0.3",
         "multiformats": "^12.1.1",
-        "rxjs": "^7.8.1"
+        "rxjs": "^7.8.1",
+        "workerpool": "^6.5.1"
     },
     "devDependencies": {
         "@polkadot/typegen": "10.9.1",
         "@types/mocha": "^10.0.2",
+        "@types/workerpool": "^6.4.5",
         "@typescript-eslint/eslint-plugin": "^6.7.3",
         "@typescript-eslint/parser": "^6.7.3",
         "mocha": "^10.2.0",

e2e/package.json

@@ -211,11 +211,6 @@ export class ExtrinsicHelper {
         return firstValueFrom(ExtrinsicHelper.api.rpc.chain.getBlock());
     }
 
-    /** engine_createBlock **/
-    public static createBlock(): Promise<CreatedBlock> {
-        return firstValueFrom(ExtrinsicHelper.api.rpc.engine.createBlock(true, true));
-    }
-
     /** Query Extrinsics */
     public static getAccountInfo(address: string): Promise<FrameSystemAccountInfo> {
         return ExtrinsicHelper.apiPromise.query.system.account(address);
@@ -243,6 +238,11 @@ export class ExtrinsicHelper {
         return new Extrinsic(() => ExtrinsicHelper.api.tx.schemas.createSchema(JSON.stringify(model), modelType, payloadLocation), keys, ExtrinsicHelper.api.events.schemas.SchemaCreated);
     }
 
+  /** Schema v2 Extrinsics */
+  public static createSchemaV2(keys: KeyringPair, model: any, modelType: "AvroBinary" | "Parquet", payloadLocation: "OnChain" | "IPFS" | "Itemized" | "Paginated", grant: ("AppendOnly"| "SignatureRequired")[]): Extrinsic {
+    return new Extrinsic(() => ExtrinsicHelper.api.tx.schemas.createSchemaV2(JSON.stringify(model), modelType, payloadLocation, grant), keys, ExtrinsicHelper.api.events.schemas.SchemaCreated);
+  }
+
     /** Generic Schema Extrinsics */
     public static createSchemaWithSettingsGov(keys: KeyringPair, model: any, modelType: "AvroBinary" | "Parquet", payloadLocation: "OnChain" | "IPFS"| "Itemized" | "Paginated", grant: "AppendOnly"| "SignatureRequired"): Extrinsic {
         return new Extrinsic(() => ExtrinsicHelper.api.tx.schemas.createSchemaViaGovernance(keys.publicKey, JSON.stringify(model), modelType, payloadLocation, [grant]), keys, ExtrinsicHelper.api.events.schemas.SchemaCreated);
@@ -425,7 +425,7 @@ export class ExtrinsicHelper {
         if (hasRelayChain()) {
             await new Promise((r) => setTimeout(r, 4_000));
         } else {
-            await ExtrinsicHelper.createBlock();
+            await firstValueFrom(ExtrinsicHelper.api.rpc.engine.createBlock(true, true));
         }
         currentBlock = await getBlockNumber();
       }

e2e/scaffolding/extrinsicHelpers.ts

@@ -1,4 +1,5 @@
 import { cryptoWaitReady } from "@polkadot/util-crypto";
+import workerpool from 'workerpool';
 import { ExtrinsicHelper } from "./extrinsicHelpers";
 import { fundingSources, getFundingSource, getRootFundingSource, getSudo } from "./funding";
 import { TEST_EPOCH_LENGTH, drainKeys, getNonce, setEpochLength } from "./helpers";
@@ -28,7 +29,7 @@ function drainAllSources() {
 }
 
 export async function mochaGlobalSetup() {
-  console.log('Global Setup Start');
+  console.log('Global Setup Start', "Reported CPU Count: ", workerpool.cpus);
   await cryptoWaitReady();
   await ExtrinsicHelper.initialize();
   await fundAllSources();

e2e/scaffolding/globalHooks.ts

@@ -313,7 +313,8 @@ describe("Delegation Scenario Tests", function () {
 
             it("revoked delegation should be reflected in all previously-granted schema permissions", async () => {
                 // Make a block first to make sure the state has rolled to the next block
-                await ExtrinsicHelper.createBlock();
+                const currentBlock = (await ExtrinsicHelper.apiPromise.rpc.chain.getBlock()).block.header.number.toNumber();
+                ExtrinsicHelper.runToBlock(currentBlock + 1);
                 const delegationsResponse = await ExtrinsicHelper.apiPromise.rpc.msa.grantedSchemaIdsByMsaId(msaId, providerId);
                 assert(delegationsResponse.isSome);
                 const delegations: SchemaGrantResponse[] = delegationsResponse.unwrap().toArray();

e2e/scenarios/grantDelegation.test.ts

@@ -26,6 +26,14 @@ describe("#createSchema", function () {
     });
   });
 
+  it("should fail if account does not have enough tokens v2", async function () {
+
+    await assert.rejects(ExtrinsicHelper.createSchemaV2(accountWithNoFunds, AVRO_GRAPH_CHANGE, "AvroBinary", "OnChain",[]).signAndSend(), {
+      name: 'RpcError',
+      message: /Inability to pay some fees/,
+    });
+  });
+
   it("should fail to create invalid schema", async function () {
     const f = ExtrinsicHelper.createSchema(keys, new Array(1000, 3), "AvroBinary", "OnChain");
 
@@ -34,13 +42,28 @@ describe("#createSchema", function () {
     });
   });
 
+  it("should fail to create invalid schema v2", async function () {
+    const f = ExtrinsicHelper.createSchemaV2(keys, new Array(1000, 3), "AvroBinary", "OnChain", []);
+
+    await assert.rejects(f.fundAndSend(fundingSource), {
+      name: 'InvalidSchema',
+    });
+  });
+
   it("should fail to create schema less than minimum size", async function () {
     const f = ExtrinsicHelper.createSchema(keys, {}, "AvroBinary", "OnChain");
     await assert.rejects(f.fundAndSend(fundingSource), {
       name: 'LessThanMinSchemaModelBytes',
     });
   });
 
+  it("should fail to create schema less than minimum size v2", async function () {
+    const f = ExtrinsicHelper.createSchemaV2(keys, {}, "AvroBinary", "OnChain", []);
+    await assert.rejects(f.fundAndSend(fundingSource), {
+      name: 'LessThanMinSchemaModelBytes',
+    });
+  });
+
   it("should fail to create schema greater than maximum size", async function () {
     const maxBytes = (await ExtrinsicHelper.getSchemaMaxBytes()).toNumber();
 
@@ -59,6 +82,24 @@ describe("#createSchema", function () {
     });
   });
 
+  it("should fail to create schema greater than maximum size v2", async function () {
+    const maxBytes = (await ExtrinsicHelper.getSchemaMaxBytes()).toNumber();
+
+    // Create a schema whose JSON representation is exactly 1 byte larger than the max allowed
+    const hugeSchema = {
+      type: "record",
+      fields: [],
+    }
+    const hugeSize = JSON.stringify(hugeSchema).length;
+    const sizeToFill = maxBytes - hugeSize - ',"name":""'.length + 1;
+    hugeSchema["name"] = Array.from(Array(sizeToFill).keys()).map(i => 'a').join('');
+
+    const f = ExtrinsicHelper.createSchemaV2(keys, hugeSchema, "AvroBinary", "OnChain", []);
+    await assert.rejects(f.fundAndSend(fundingSource), {
+      name: 'ExceedsMaxSchemaModelBytes',
+    });
+  });
+
   it("should successfully create an Avro GraphChange schema", async function () {
     const f = ExtrinsicHelper.createSchema(keys, AVRO_GRAPH_CHANGE, "AvroBinary", "OnChain");
     const [createSchemaEvent, eventMap] = await f.fundAndSend(fundingSource);
@@ -67,4 +108,12 @@ describe("#createSchema", function () {
     assert.notEqual(createSchemaEvent, undefined);
   });
 
+  it("should successfully create an Avro GraphChange schema v2", async function () {
+    const f = ExtrinsicHelper.createSchemaV2(keys, AVRO_GRAPH_CHANGE, "AvroBinary", "OnChain", []);
+    const [createSchemaEvent, eventMap] = await f.fundAndSend(fundingSource);
+
+    assertExtrinsicSuccess(eventMap);
+    assert.notEqual(createSchemaEvent, undefined);
+  });
+
 })

e2e/schemas/createSchema.test.ts

@@ -13,7 +13,6 @@ use frame_support::{assert_ok, pallet_prelude::DispatchResult};
 use frame_system::{pallet_prelude::BlockNumberFor, RawOrigin};
 use sp_runtime::traits::One;
 
-const AVERAGE_NUMBER_OF_MESSAGES: u32 = 499;
 const IPFS_SCHEMA_ID: u16 = 50;
 const IPFS_PAYLOAD_LENGTH: u32 = 10;
 
@@ -23,7 +22,7 @@ fn onchain_message<T: Config>(schema_id: SchemaId) -> DispatchResult {
 	let payload = Vec::from(
 		"{'fromId': 123, 'content': '232323', 'fromId': 123, 'content': '232323'}".as_bytes(),
 	);
-	let bounded_payload: BoundedVec<u8, T::MaxMessagePayloadSizeBytes> =
+	let bounded_payload: BoundedVec<u8, T::MessagesMaxPayloadSizeBytes> =
 		payload.try_into().expect("Invalid payload");
 	MessagesPallet::<T>::add_message(
 		provider_id.into(),
@@ -40,7 +39,7 @@ fn ipfs_message<T: Config>(schema_id: SchemaId) -> DispatchResult {
 	let payload =
 		Vec::from("bafkreidgvpkjawlxz6sffxzwgooowe5yt7i6wsyg236mfoks77nywkptdq".as_bytes());
 	let provider_id = ProviderId(1);
-	let bounded_payload: BoundedVec<u8, T::MaxMessagePayloadSizeBytes> =
+	let bounded_payload: BoundedVec<u8, T::MessagesMaxPayloadSizeBytes> =
 		payload.try_into().expect("Invalid payload");
 
 	MessagesPallet::<T>::add_message(
@@ -63,8 +62,10 @@ fn create_schema<T: Config>(location: PayloadLocation) -> DispatchResult {
 }
 
 benchmarks! {
+	// this is temporary to avoid massive PoV sizes which will break the chain until rework on messages
+	#[pov_mode = Measured]
 	add_onchain_message {
-		let n in 0 .. T::MaxMessagePayloadSizeBytes::get() - 1;
+		let n in 0 .. T::MessagesMaxPayloadSizeBytes::get() - 1;
 		let message_source_id = DelegatorId(2);
 		let caller: T::AccountId = whitelisted_caller();
 		let schema_id = 1;
@@ -77,19 +78,21 @@ benchmarks! {
 		assert_ok!(T::MsaBenchmarkHelper::set_delegation_relationship(ProviderId(1), message_source_id.into(), [schema_id].to_vec()));
 
 		let payload = vec![1; n as usize];
-
-		for j in 1 .. AVERAGE_NUMBER_OF_MESSAGES {
+		let average_messages_per_block: u32 = T::MaxMessagesPerBlock::get() / 2;
+		for j in 1 .. average_messages_per_block {
 			assert_ok!(onchain_message::<T>(schema_id));
 		}
 	}: _ (RawOrigin::Signed(caller), Some(message_source_id.into()), schema_id, payload)
 	verify {
 		assert_eq!(
 			MessagesPallet::<T>::get_messages(
 				BlockNumberFor::<T>::one(), schema_id).len(),
-			AVERAGE_NUMBER_OF_MESSAGES as usize
+			average_messages_per_block as usize
 		);
 	}
 
+	// this is temporary to avoid massive PoV sizes which will break the chain until rework on messages
+	#[pov_mode = Measured]
 	add_ipfs_message {
 		let caller: T::AccountId = whitelisted_caller();
 		let cid = "bafkreidgvpkjawlxz6sffxzwgooowe5yt7i6wsyg236mfoks77nywkptdq".as_bytes().to_vec();
@@ -99,16 +102,16 @@ benchmarks! {
 			assert_ok!(create_schema::<T>(PayloadLocation::IPFS));
 		}
 		assert_ok!(T::MsaBenchmarkHelper::add_key(ProviderId(1).into(), caller.clone()));
-
-		for j in 1 .. AVERAGE_NUMBER_OF_MESSAGES {
+		let average_messages_per_block: u32 = T::MaxMessagesPerBlock::get() / 2;
+		for j in 1 .. average_messages_per_block {
 			assert_ok!(ipfs_message::<T>(IPFS_SCHEMA_ID));
 		}
 	}: _ (RawOrigin::Signed(caller),IPFS_SCHEMA_ID, cid, IPFS_PAYLOAD_LENGTH)
 	verify {
 		assert_eq!(
 			MessagesPallet::<T>::get_messages(
 				BlockNumberFor::<T>::one(), IPFS_SCHEMA_ID).len(),
-			AVERAGE_NUMBER_OF_MESSAGES as usize
+			average_messages_per_block as usize
 		);
 	}