create new canisters on demand; deploy stored wasm (#264)

* create new canisters on demand; deploy stored wasm * fix * fix * fix * chore: Whitelist frontend canister hash from dfx version 0.24.1 (#265) Co-authored-by: adamspofford-dfinity <[email protected]> * fix * fix * fix cycle after expiration (#266) * fix * fix * fix --------- Co-authored-by: DFINITY bot <[email protected]> Co-authored-by: adamspofford-dfinity <[email protected]>
dfinity · Oct 30, 2024 · 1275ac8 · 1275ac8
1 parent e41aeb7
commit 1275ac8
Show file tree

Hide file tree

Showing 7 changed files with 128 additions and 72 deletions.
diff --git a/service/pool/Main.mo b/service/pool/Main.mo
@@ -84,7 +84,7 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
         await* f2;
     };
     private func getExpiredCanisterInfo(origin : Logs.Origin) : async* (Types.CanisterInfo, {#install; #reinstall}) {
-        switch (pool.getExpiredCanisterId()) {
+        let res = switch (pool.getExpiredCanisterId()) {
             case (#newId) {
               try {
                 Cycles.add<system>(params.cycles_per_canister);
@@ -109,7 +109,6 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
               };
             };
             case (#reuse info) {
-                let no_uninstall = Option.get(params.no_uninstall, false);
                 let cid = { canister_id = info.id };
                 let status = await IC.canister_status cid;
                 let topUpCycles : Nat = if (status.cycles < params.cycles_per_canister) {
@@ -119,7 +118,12 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
                     Cycles.add<system> topUpCycles;
                     await IC.deposit_cycles cid;
                 };
-                if (not no_uninstall and Option.isSome(status.module_hash)) {
+                let need_uninstall = switch ((params.stored_module, status.module_hash)) {
+                case ((null, ?_)) { true };
+                case ((_, null)) { false };
+                case (?stored, ?current) { stored.hash != current };
+                };
+                if (need_uninstall) {
                     await* pool_uninstall_code(cid.canister_id);
                 };
                 switch (status.status) {
@@ -130,7 +134,7 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
                 };
                 stats := Logs.updateStats(stats, #getId topUpCycles);
                 statsByOrigin.addCanister(origin);
-                let mode = if (no_uninstall) { #reinstall } else { #install };
+                let mode = if (need_uninstall and Option.isNull(params.stored_module)) { #install } else { #reinstall };
                 (info, mode);
             };
             case (#outOfCapacity time) {
@@ -139,6 +143,20 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
                 throw Error.reject("No available canister id, wait for " # debug_show (second) # " seconds.");
             };
         };
+        switch (params.stored_module) {
+        case null {};
+        case (?stored) {
+                 await IC.install_chunked_code {
+                     arg = stored.arg;
+                     target_canister = res.0.id;
+                     store_canister = ?(Principal.fromActor this);
+                     chunk_hashes_list = [{ hash = stored.hash }];
+                     wasm_module_hash = stored.hash;
+                     mode = res.1;
+                 }
+             };
+        };
+        res;
     };
     func validateOrigin(origin: Logs.Origin) : Bool {
         if (origin.origin == "") {
@@ -192,6 +210,9 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
         if (not Principal.isController(caller)) {
             throw Error.reject "Only called by controller";
         };
+        if (Option.isSome(params.stored_module) and Option.isSome(args)) {
+            throw Error.reject "args should be null when stored_module is set";
+        };
         let origin = { origin = "admin"; tags = [] };
         let (info, mode) = switch (opt_info) {
         case null { await* getExpiredCanisterInfo(origin) };
@@ -242,6 +263,9 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
     };
 
     public shared ({ caller }) func getCanisterId(nonce : PoW.Nonce, origin : Logs.Origin) : async Types.CanisterInfo {
+        if (Option.get(params.admin_only, false)) {
+            throw Error.reject "Cannot call this endpoint when admin_only is true";
+        };
         if (not validateOrigin(origin)) {
             throw Error.reject "Please specify a valid origin";
         };
@@ -358,7 +382,7 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
     };
 
     func updateTimer<system>(info: Types.CanisterInfo) {
-        if (Option.get(params.no_uninstall, false)) {
+        if (Option.isSome(params.stored_module)) {
             return;
         };
         func job() : async () {
@@ -436,7 +460,7 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
             throw Error.reject "only called by controllers";
         };
         for (info in pool.getAllCanisters()) {
-            if (not Option.get(params.no_uninstall, false)) {
+            if (Option.isNull(params.stored_module)) {
                 await* pool_uninstall_code(info.id);
             };
             ignore pool.retire info;

diff --git a/service/pool/Types.mo b/service/pool/Types.mo
@@ -20,8 +20,9 @@ module {
         canister_time_to_live: Nat;
         nonce_time_to_live: Nat;
         max_family_tree_size: Nat;
-        // Used for asset canister. If set to true, will not use timer to kill expired canisters, and will not uninstall code when fetching an expired canister.
-        no_uninstall: ?Bool;
+        // Used for installing asset canister. If set, will not use timer to kill expired canisters, and will not uninstall code when fetching an expired canister (unless the module hash changed).
+        stored_module: ?{hash: Blob; arg: Blob};
+        admin_only: ?Bool;
         wasm_utils_principal: ?Text;
     };
     public let defaultParams : InitParams = {
@@ -30,7 +31,8 @@ module {
         canister_time_to_live = 1200_000_000_000;
         nonce_time_to_live = 300_000_000_000;
         max_family_tree_size = 5;
-        no_uninstall = null;
+        stored_module = null;
+        admin_only = null;
         wasm_utils_principal = ?"ozk6r-tyaaa-aaaab-qab4a-cai";
     };
     public type InstallArgs = {
@@ -90,29 +92,35 @@ module {
             len -= 1;
         };
         public func getExpiredCanisterId() : NewId {
-            if (len < size) {
-                // increment len here to prevent race condition
-                len += 1;
-                #newId
-            } else {
-                switch (tree.entries().next()) {
-                    case null { assert false; loop(); };
-                    case (?info) {
-                        let now = Time.now();
-                        let elapsed : Nat = Int.abs(now) - Int.abs(info.timestamp);
-                        if (elapsed >= ttl) {
-                            // Lazily cleanup pool state before reusing canister
-                            tree.remove info;
-                            let newInfo = { timestamp = now; id = info.id; };
-                            tree.insert newInfo;
-                            metadata.put(newInfo.id, (newInfo.timestamp, false));
-                            deleteFamilyNode(newInfo.id);
-                            #reuse newInfo
-                        } else {
-                            #outOfCapacity(ttl - elapsed)
-                        }
+            switch (tree.entries().next()) {
+            case null {
+                     if (len < size) {
+                         len += 1;
+                         #newId
+                     } else {
+                         Debug.trap "No canister in the pool"
                      };
-                };
+                 };
+            case (?info) {
+                     let now = Time.now();
+                     let elapsed : Nat = Int.abs(now) - Int.abs(info.timestamp);
+                     if (elapsed >= ttl) {
+                         // Lazily cleanup pool state before reusing canister
+                         tree.remove info;
+                         let newInfo = { timestamp = now; id = info.id; };
+                         tree.insert newInfo;
+                         metadata.put(newInfo.id, (newInfo.timestamp, false));
+                         deleteFamilyNode(newInfo.id);
+                         #reuse newInfo
+                     } else {
+                         if (len < size) {
+                             len += 1;
+                             #newId
+                         } else {
+                             #outOfCapacity(ttl - elapsed)
+                         }
+                     }
+                 };
             };
         };
         public func removeCanister(info: CanisterInfo) {
@@ -168,6 +176,8 @@ module {
             tree.insert { timestamp = 0; id };
             metadata.put(id, (0, false));
             deleteFamilyNode id;
+            cycles.delete id;
+            // snapshots already removed with pool_uninstall_code
             return true;
         };
 

diff --git a/service/pool/tests/assetstorage.wasm.gz b/service/pool/tests/assetstorage.wasm.gz
diff --git a/service/pool/tests/canisterPool.test.sh b/service/pool/tests/canisterPool.test.sh
@@ -11,26 +11,7 @@ let init = opt record {
   nonce_time_to_live = 1;
   canister_time_to_live = 5_000_000_000;
   max_family_tree_size = 5;
-  no_uninstall = opt true;
-};
-let S = install(wasm, init, null);
-let nonce = record { timestamp = 1 : int; nonce = 1 : nat };
-let CID2 = call S.getCanisterId(nonce, origin);
-call S.installCode(CID2, record { arg = blob ""; wasm_module = empty_wasm; mode = variant { install }; canister_id = CID2.id }, record { profiling = false; is_whitelisted = false; origin = origin });
-read_state("canister", CID2.id, "module_hash");
-let c1 = call S.deployCanister(null, opt record { arg = blob ""; wasm_module = empty_wasm; bypass_wasm_transform = opt true });
-let c1 = c1[0];
-call S.transferOwnership(c1, vec {c1.id; S});
-fail call S.deployCanister(opt c1, opt record { arg = blob ""; wasm_module = empty_wasm; bypass_wasm_transform = opt true });
-assert _ ~= "Cannot find canister";
-
-let init = opt record {
-  cycles_per_canister = 105_000_000_000;
-  max_num_canisters = 2;
-  nonce_time_to_live = 1;
-  canister_time_to_live = 5_000_000_000;
-  max_family_tree_size = 5;
-  no_uninstall = opt false;
+  stored_module = null;
 };
 let S = install(wasm, init, null);
 let nonce = record { timestamp = 1 : int; nonce = 1 : nat };
@@ -51,17 +32,11 @@ let init = opt record {
 };
 let S = install(wasm, init, null);
 
-let c1 = call S.getCanisterId(nonce, origin);
-c1;
-let c2 = call S.getCanisterId(nonce, origin);
-c2;
-let c3 = call S.getCanisterId(nonce, origin);
-c3;
-let c4 = call S.getCanisterId(nonce, origin);
-c4;
-assert c1.id != c2.id;
-assert c1.id == c3.id;
-assert c2.id == c4.id;
+let s1 = par_call [S.getCanisterId(nonce, origin), S.getCanisterId(nonce, origin)];
+assert s1[0].id != s1[1].id;
+let s2 = par_call [S.getCanisterId(nonce, origin), S.getCanisterId(nonce, origin)];
+assert or(eq(s1[0].id, s2[0].id), eq(s1[0].id, s2[1].id)) == true;
+assert or(eq(s1[1].id, s2[1].id), eq(s1[1].id, s2[0].id)) == true;
 
 // Out of capacity
 let init = opt record {
@@ -105,5 +80,4 @@ call S.getCanisterId(nonce, origin);
 // Enough time has passed that the timer has removed the canister code
 fail read_state("canister", CID.id, "module_hash");
 assert _ ~= "absent";
-read_state("canister", CID2.id, "module_hash");
 read_state("canister", CID3.id, "module_hash");
diff --git a/service/pool/tests/frontend.test.sh b/service/pool/tests/frontend.test.sh
@@ -0,0 +1,50 @@
+#!ic-repl
+load "prelude.sh";
+
+let wasm = file("../../../.dfx/local/canisters/backend/backend.wasm");
+let asset = file("assetstorage.wasm.gz");
+let module_hash = blob "\29\6d\1a\d1\a7\f8\b1\5f\90\ff\8b\72\86\58\64\6b\64\9c\ab\d1\59\f3\60\f1\b4\27\29\7f\4c\76\76\3e";
+let empty_wasm = blob "\00asm\01\00\00\00";
+
+let origin = record { origin = "test"; tags = vec {"tag"} };
+let nonce = record { timestamp = 1 : int; nonce = 1 : nat };
+let init = opt record {
+  cycles_per_canister = 105_000_000_000;
+  max_num_canisters = 2;
+  nonce_time_to_live = 1;
+  canister_time_to_live = 3_000_000_000;
+  max_family_tree_size = 5;
+  stored_module = opt record { hash = module_hash; arg = encode () };
+};
+let S = install(wasm, init, null);
+call ic.upload_chunk(record { chunk = asset; canister_id = S });
+let nonce = record { timestamp = 1 : int; nonce = 1 : nat };
+let c1 = call S.deployCanister(null, null);
+let c1 = c1[0];
+call S.transferOwnership(c1, vec {c1.id; S});
+fail call S.deployCanister(opt c1, null);
+assert _ ~= "Cannot find canister";
+let c2 = call S.getCanisterId(nonce, origin);
+let c3 = call S.deployCanister(null, null);
+assert read_state("canister", c2.id, "module_hash") == module_hash;
+assert c2.id != c1.id;
+assert c3[0].id != c2.id;
+
+call ic.upload_chunk(record { chunk = empty_wasm; canister_id = S });
+let hash = _.hash;
+let init = opt record {
+  cycles_per_canister = 105_000_000_000;
+  max_num_canisters = 2;
+  nonce_time_to_live = 1;
+  canister_time_to_live = 3_000_000_000;
+  max_family_tree_size = 5;
+  stored_module = opt record { hash = hash; arg = encode () };
+};
+upgrade(S, wasm, init);
+let c4 = call S.getCanisterId(nonce, origin);
+assert c4.id == c2.id;
+
+assert read_state("canister", c1.id, "module_hash") == module_hash;
+assert read_state("canister", c3[0].id, "module_hash") == module_hash;
+assert read_state("canister", c4.id, "module_hash") == hash;
+
diff --git a/service/pool/tests/upgrade.test.sh b/service/pool/tests/upgrade.test.sh
@@ -15,16 +15,12 @@ let S = install(wasm, init, null);
 
 let nonce = record { timestamp = 1 : int; nonce = 1 : nat };
 let c1 = call S.getCanisterId(nonce, origin);
-c1;
 let c2 = call S.getCanisterId(nonce, origin);
-c2;
 
 upgrade(S, wasm, init);
 let c3 = call S.getCanisterId(nonce, origin);
-c3;
 let c4 = call S.getCanisterId(nonce, origin);
-c4;
-assert c1.id != c2.id;
+assert c1.id == c2.id;
 assert c1.id == c3.id;
 assert c2.id == c4.id;
 
@@ -41,10 +37,11 @@ upgrade(S, wasm, init);
 // stats are preserved after upgrade
 call S.getStats();
 assert _ == stats;
-let c5 = call S.getCanisterId(nonce, origin);
-c5;
-assert c5.id != c1.id;
-assert c5.id != c2.id;
+// TTL increased, c4 suddenly get more time.
+let c6 = call S.getCanisterId(nonce, origin);
+let c7 = call S.getCanisterId(nonce, origin);
+assert c6.id != c4.id;
+assert c7.id != c6.id;
 fail call S.getCanisterId(nonce, origin);
 assert _ ~= "No available canister id";
 

diff --git a/service/wasm-utils/whitelisted_wasms.txt b/service/wasm-utils/whitelisted_wasms.txt
@@ -8,4 +8,5 @@
     "32e92f1190d8321e97f8d8f3e793019e4fd2812bfc595345d46d2c23f74c1ab5", // dfx 0.18.0 -- 0.20.1 frontend canister
     "2cc4ec4381dee231379270a08403c984986c9fc0c2eaadb64488b704a3104cc0", // dfx 0.21.0 frontend canister
     "3a533f511b3960b4186e76cf9abfbd8222a2c507456a66ec55671204ee70cae3", // dfx 0.23.0 frontend canister
+    "2c9e30df9be951a6884c702a97bbb8c0b438f33d4208fa612b1de6fb1752db76", // dfx 0.24.1 frontend canister
 ]