Skip to content

Commit

Permalink
Add stats by origin (#192)
Browse files Browse the repository at this point in the history 
* add stats by origin

* fix

* fix

* frontend

* fix

* Dynamically change installer origin when loading projects from various sources (#193)

* Convert 'allowedOrigins' to TypeScript

* Add 'setOrigin' workplace reducer action

* Progress

* Refactor example projects to use example name in origin

* Misc

* Use URL origin in place of editor key for 'playground:post'

* Reintroduce details for 'tag' and 'git' origins

* Keep track of 'document.referrer' and 'window.opener'

* Misc

* change origin type to include array of tags

* fix

* add backend tags

---------

Co-authored-by: Ryan Vandersmith <[email protected]>
  • Loading branch information
@chenyan-dfinity @rvanasa
chenyan-dfinity and rvanasa authored Sep 8, 2023
1 parent 3f41a9a commit 856f44f
Show file tree
Hide file tree
Showing 25 changed files with 428 additions and 114 deletions.
82 changes: 82 additions & 0 deletions service/pool/Logs.mo
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,86 @@
import Map "mo:base/RBTree";
import {compare} "mo:base/Text";
import {toArray} "mo:base/Iter";
import {now = timeNow} "mo:base/Time";
import {toText} "mo:base/Int";
import {get} "mo:base/Option";

module {
public type Origin = { origin: Text; tags: [Text] };
public type SharedStatsByOrigin = (Map.Tree<Text,Nat>, Map.Tree<Text,Nat>, Map.Tree<Text, Nat>);
public class StatsByOrigin() {
var canisters = Map.RBTree<Text, Nat>(compare);
var installs = Map.RBTree<Text, Nat>(compare);
var tags = Map.RBTree<Text, Nat>(compare);
public func share() : SharedStatsByOrigin = (canisters.share(), installs.share(), tags.share());
public func unshare(x : SharedStatsByOrigin) {
canisters.unshare(x.0);
installs.unshare(x.1);
tags.unshare(x.2);
};
func addTags(list: [Text]) {
for (tag in list.vals()) {
switch (tags.get(tag)) {
case null { tags.put(tag, 1) };
case (?n) { tags.put(tag, n + 1) };
};
};
};
public func addCanister(origin: Origin) {
switch (canisters.get(origin.origin)) {
case null { canisters.put(origin.origin, 1) };
case (?n) { canisters.put(origin.origin, n + 1) };
};
// Not storing tags for create canister to avoid duplicate counting of tags
// addTags(origin.tags);
};
public func addInstall(origin: Origin) {
switch (installs.get(origin.origin)) {
case null { installs.put(origin.origin, 1) };
case (?n) { installs.put(origin.origin, n + 1) };
};
// Only record tags for canister install
addTags(origin.tags);
};
public func dump() : ([(Text, Nat)], [(Text, Nat)], [(Text, Nat)]) {
(toArray<(Text, Nat)>(canisters.entries()),
toArray<(Text, Nat)>(installs.entries()),
toArray<(Text, Nat)>(tags.entries())
)
};
public func metrics() : Text {
var result = "";
let now = timeNow() / 1_000_000;
for ((origin, count) in canisters.entries()) {
let name = "canisters_" # origin;
let desc = "Number of canisters requested from " # origin;
result := result # encode_single_value("counter", name, count, desc, now);
};
for ((origin, count) in installs.entries()) {
let name = "installs_" # origin;
let desc = "Number of Wasm installed from " # origin;
result := result # encode_single_value("counter", name, count, desc, now);
};
let profiling = get(tags.get("profiling"), 0);
let asset = get(tags.get("asset"), 0);
let install = get(tags.get("install"), 0);
let reinstall = get(tags.get("reinstall"), 0);
let upgrade = get(tags.get("upgrade"), 0);
result := result
# encode_single_value("counter", "profiling", profiling, "Number of Wasm profiled", now)
# encode_single_value("counter", "asset", asset, "Number of asset Wasm canister installed", now)
# encode_single_value("counter", "install", install, "Number of Wasm with install mode", now)
# encode_single_value("counter", "reinstall", reinstall, "Number of Wasm with reinstall mode", now)
# encode_single_value("counter", "upgrade", upgrade, "Number of Wasm with upgrad mode", now);
result;
};
};
public func encode_single_value(kind: Text, name: Text, number: Int, desc: Text, time: Int) : Text {
"# HELP " # name # " " # desc # "\n" #
"# TYPE " # name # " " # kind # "\n" #
name # " " # toText(number) # " " # toText(time) # "\n"
};

public type Stats = {
num_of_canisters: Nat;
num_of_installs: Nat;
Expand Down
61 changes: 47 additions & 14 deletions service/pool/Main.mo
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import Option "mo:base/Option";
import Nat "mo:base/Nat";
import Text "mo:base/Text";
import Array "mo:base/Array";
import Buffer "mo:base/Buffer";
import List "mo:base/List";
import Deque "mo:base/Deque";
import Result "mo:base/Result";
Expand All @@ -23,6 +24,7 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
let params = Option.get(opt_params, Types.defaultParams);
var pool = Types.CanisterPool(params.max_num_canisters, params.canister_time_to_live, params.max_family_tree_size);
let nonceCache = PoW.NonceCache(params.nonce_time_to_live);
var statsByOrigin = Logs.StatsByOrigin();

stable let controller = creator.caller;
stable var stats = Logs.defaultStats;
Expand All @@ -31,6 +33,7 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
stable var stableChildren : [(Principal, [Principal])] = [];
stable var stableTimers : [Types.CanisterInfo] = [];
stable var previousParam : ?Types.InitParams = null;
stable var stableStatsByOrigin : Logs.SharedStatsByOrigin = (#leaf, #leaf, #leaf);

system func preupgrade() {
let (tree, metadata, children, timers) = pool.share();
Expand All @@ -39,6 +42,7 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
stableChildren := children;
stableTimers := timers;
previousParam := ?params;
stableStatsByOrigin := statsByOrigin.share();
};

system func postupgrade() {
Expand All @@ -50,15 +54,17 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
pool.unshare(stablePool, stableMetadata, stableChildren);
for (info in stableTimers.vals()) {
updateTimer(info);
}
};
statsByOrigin.unshare(stableStatsByOrigin);
};

public query func getInitParams() : async Types.InitParams {
params;
};

public query func getStats() : async Logs.Stats {
stats;
public query func getStats() : async (Logs.Stats, [(Text, Nat)], [(Text, Nat)], [(Text, Nat)]) {
let (canister, install, tags) = statsByOrigin.dump();
(stats, canister, install, tags);
};

public query func balance() : async Nat {
Expand All @@ -70,7 +76,7 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
ignore Cycles.accept amount;
};

private func getExpiredCanisterInfo() : async Types.CanisterInfo {
private func getExpiredCanisterInfo(origin : Logs.Origin) : async Types.CanisterInfo {
switch (pool.getExpiredCanisterId()) {
case (#newId) {
Cycles.add(params.cycles_per_canister);
Expand All @@ -79,6 +85,7 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
let info = { id = cid.canister_id; timestamp = now };
pool.add info;
stats := Logs.updateStats(stats, #getId(params.cycles_per_canister));
statsByOrigin.addCanister(origin);
info;
};
case (#reuse info) {
Expand All @@ -101,6 +108,7 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
case _ {};
};
stats := Logs.updateStats(stats, #getId topUpCycles);
statsByOrigin.addCanister(origin);
info;
};
case (#outOfCapacity time) {
Expand All @@ -111,7 +119,10 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
};
};

public shared ({ caller }) func getCanisterId(nonce : PoW.Nonce) : async Types.CanisterInfo {
public shared ({ caller }) func getCanisterId(nonce : PoW.Nonce, origin : Logs.Origin) : async Types.CanisterInfo {
if (origin.origin == "") {
throw Error.reject "Please specify an origin";
};
if (caller != controller and not nonceCache.checkProofOfWork(nonce)) {
stats := Logs.updateStats(stats, #mismatch);
throw Error.reject "Proof of work check failed";
Expand All @@ -122,10 +133,14 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
throw Error.reject "Nonce already used";
};
nonceCache.add nonce;
await getExpiredCanisterInfo();
await getExpiredCanisterInfo(origin);
};

public shared ({ caller }) func installCode(info : Types.CanisterInfo, args : Types.InstallArgs, profiling : Bool, is_whitelisted : Bool) : async Types.CanisterInfo {
type InstallConfig = { profiling: Bool; is_whitelisted: Bool; origin: Logs.Origin };
public shared ({ caller }) func installCode(info : Types.CanisterInfo, args : Types.InstallArgs, install_config : InstallConfig) : async Types.CanisterInfo {
if (install_config.origin.origin == "") {
throw Error.reject "Please specify an origin";
};
if (info.timestamp == 0) {
stats := Logs.updateStats(stats, #mismatch);
throw Error.reject "Cannot install removed canister";
Expand All @@ -135,14 +150,14 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
throw Error.reject "Cannot find canister";
} else {
let config = {
profiling;
profiling = install_config.profiling;
remove_cycles_add = true;
limit_stable_memory_page = ?(16384 : Nat32); // Limit to 1G of stable memory
backend_canister_id = ?Principal.fromActor(this);
};
let wasm = if (caller == controller and is_whitelisted) {
let wasm = if (caller == controller and install_config.is_whitelisted) {
args.wasm_module;
} else if (is_whitelisted) {
} else if (install_config.is_whitelisted) {
await Wasm.is_whitelisted(args.wasm_module);
} else {
await Wasm.transform(args.wasm_module, config);
Expand All @@ -155,7 +170,23 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
};
await IC.install_code newArgs;
stats := Logs.updateStats(stats, #install);
switch (pool.refresh(info, profiling)) {

// Build tags from install arguments
let tags = Buffer.fromArray<Text>(install_config.origin.tags);
if (install_config.profiling) {
tags.add("profiling");
};
if (install_config.is_whitelisted) {
tags.add("asset");
};
switch (args.mode) {
case (#install) { tags.add("install") };
case (#upgrade) { tags.add("upgrade") };
case (#reinstall) { tags.add("reinstall") };
};
let origin = { origin = install_config.origin.origin; tags = Buffer.toArray(tags) };
statsByOrigin.addInstall(origin);
switch (pool.refresh(info, install_config.profiling)) {
case (?newInfo) {
updateTimer(newInfo);
newInfo;
Expand Down Expand Up @@ -241,12 +272,13 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
throw Error.reject "Only called by controller";
};
stats := Logs.defaultStats;
statsByOrigin := Logs.StatsByOrigin();
};

// Metrics
public query func http_request(req : Metrics.HttpRequest) : async Metrics.HttpResponse {
if (req.url == "/metrics") {
let body = Metrics.metrics stats;
let body = Metrics.metrics(stats, statsByOrigin);
{
status_code = 200;
headers = [("Content-Type", "text/plain; version=0.0.4"), ("Content-Length", Nat.toText(body.size()))];
Expand Down Expand Up @@ -294,7 +326,7 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
if (not pool.findId caller) {
throw Error.reject "Only a canister managed by the Motoko Playground can call create_canister";
};
let info = await getExpiredCanisterInfo();
let info = await getExpiredCanisterInfo({origin="spawned"; tags=[]});
let result = pool.setChild(caller, info.id);
if (not result) {
throw Error.reject("In the Motoko Playground, each top level canister can only spawn " # Nat.toText(params.max_family_tree_size) # " descendants including itself");
Expand All @@ -319,7 +351,8 @@ shared (creator) actor class Self(opt_params : ?Types.InitParams) = this {
switch (sanitizeInputs(caller, canister_id)) {
case (#ok info) {
let args = { arg; wasm_module; mode; canister_id };
ignore await installCode(info, args, pool.profiling caller, false); // inherit the profiling of the parent
let config = { profiling = pool.profiling caller; is_whitelisted = false; origin = {origin = "spawned"; tags = [] } };
ignore await installCode(info, args, config); // inherit the profiling of the parent
};
case (#err makeMsg) throw Error.reject(makeMsg "install_code");
};
Expand Down
10 changes: 3 additions & 7 deletions service/pool/Metrics.mo
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
import Text "mo:base/Text";
import Time "mo:base/Time";
import Int "mo:base/Int";
import Logs "./Logs";

module {
Expand All @@ -15,12 +14,8 @@ module {
headers: [(Text, Text)];
body: Blob;
};
func encode_single_value(kind: Text, name: Text, number: Int, desc: Text, time: Int) : Text {
"# HELP " # name # " " # desc # "\n" #
"# TYPE " # name # " " # kind # "\n" #
name # " " # Int.toText(number) # " " # Int.toText(time) # "\n"
};
public func metrics(stats: Logs.Stats) : Blob {
let encode_single_value = Logs.encode_single_value;
public func metrics(stats: Logs.Stats, origin: Logs.StatsByOrigin) : Blob {
let now = Time.now() / 1_000_000;
var result = "";
result := result # encode_single_value("counter", "canister_count", stats.num_of_canisters, "Number of canisters deployed", now);
Expand All @@ -29,6 +24,7 @@ module {
result := result # encode_single_value("counter", "out_of_capacity", stats.error_out_of_capacity, "Number of out of capacity requests", now);
result := result # encode_single_value("counter", "total_wait_time", stats.error_total_wait_time, "Number of seconds waiting for out of capacity requests", now);
result := result # encode_single_value("counter", "mismatch", stats.error_mismatch, "Number of mismatch requests including wrong nounce and timestamp", now);
result := result # origin.metrics();
Text.encodeUtf8(result)
};
}
21 changes: 11 additions & 10 deletions service/pool/tests/actor_class/test.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,10 @@ let deleter = file(".dfx/local/canisters/Deleter/Deleter.wasm");
let S = install(wasm, null, opt 100_000_000_000_000);

let nonce = record { timestamp = 1 : int; nonce = 1 : nat };
let c1 = call S.getCanisterId(nonce);
let origin = record { origin = "test"; tags = vec {} };
let c1 = call S.getCanisterId(nonce, origin);
let args = record { arg = blob ""; wasm_module = parent; mode = variant { install }; canister_id = c1.id };
call S.installCode(c1, args, false, false);
call S.installCode(c1, args, record { profiling = false; is_whitelisted = false; origin = origin });

let c1 = c1.id;

Expand Down Expand Up @@ -48,26 +49,26 @@ let init = opt record {
let S = install(wasm, init, opt 100_000_000_000_000);

let nonce = record { timestamp = 1 : int; nonce = 1 : nat };
let c1 = call S.getCanisterId(nonce);
let c1 = call S.getCanisterId(nonce, origin);
let args = record { arg = blob ""; wasm_module = parent; mode = variant { install }; canister_id = c1.id };
call S.installCode(c1, args, false, false);
call S.installCode(c1, args, record { profiling = false; is_whitelisted = false; origin = origin });
let c1 = c1.id;

fail call c1.makeChild(0);
call S.getCanisterId(nonce);
call S.getCanisterId(nonce);
call S.getCanisterId(nonce, origin);
call S.getCanisterId(nonce, origin);

// Security check
let S = install(wasm, null, opt 100_000_000_000_000);

let nonce = record { timestamp = 1 : int; nonce = 1 : nat };
let c1 = call S.getCanisterId(nonce);
let c1 = call S.getCanisterId(nonce, origin);
let args = record { arg = blob ""; wasm_module = parent; mode = variant { install }; canister_id = c1.id };
call S.installCode(c1, args, false, false);
call S.installCode(c1, args, record { profiling = false; is_whitelisted = false; origin = origin });

let c2 = call S.getCanisterId(nonce);
let c2 = call S.getCanisterId(nonce, origin);
let args = record { arg = blob ""; wasm_module = deleter; mode = variant { install }; canister_id = c2.id };
call S.installCode(c2, args, false, false);
call S.installCode(c2, args, record { profiling = false; is_whitelisted = false; origin = origin });

let c1 = c1.id;
let c2 = c2.id;
Expand Down
Loading

0 comments on commit 856f44f

Please sign in to comment.