diff --git a/examples/snake/ws-resources.json b/examples/snake/ws-resources.json index 2b67ce4f..95c51310 100644 --- a/examples/snake/ws-resources.json +++ b/examples/snake/ws-resources.json @@ -4,5 +4,8 @@ "Content-Type": "text/html; charset=utf-8", "Cache-Control": "max-age=3500" } + }, + "routes": { + "/path/*": "/file.svg" } } diff --git a/move/walrus_site/Move.lock b/move/walrus_site/Move.lock index 16e441f3..ef67a42d 100644 --- a/move/walrus_site/Move.lock +++ b/move/walrus_site/Move.lock @@ -2,7 +2,7 @@ [move] version = 0 -manifest_digest = "E9C7401E4D5BF8E549B5CB4E935992B4BEEEB50757B900BB03D72491F31A7087" +manifest_digest = "6955A2973AF0DB196E469AD72FDB6D203278DF6207C4561B7B9FDC4BD1AC951D" deps_digest = "F8BBB0CCB2491CA29A3DF03D6F92277A4F3574266507ACD77214D37ECA3F3082" dependencies = [ @@ -22,7 +22,7 @@ dependencies = [ ] [move.toolchain-version] -compiler-version = "1.33.1" +compiler-version = "1.34.2" edition = "2024.beta" flavor = "sui" @@ -36,6 +36,6 @@ published-version = "1" [env.testnet] chain-id = "4c78adac" -original-published-id = "0xe15cd956d3f54ad0b6608b01b96e9999d66552dfd025e698ac16cd0df1787a25" -latest-published-id = "0xe15cd956d3f54ad0b6608b01b96e9999d66552dfd025e698ac16cd0df1787a25" +original-published-id = "0x8a928f3cb7b4b42a957d554acd3928c4fbf5b0052f8218513a259f03837390fc" +latest-published-id = "0x8a928f3cb7b4b42a957d554acd3928c4fbf5b0052f8218513a259f03837390fc" published-version = "1" diff --git a/move/walrus_site/Move.toml b/move/walrus_site/Move.toml index 32d7525e..3b481797 100644 --- a/move/walrus_site/Move.toml +++ b/move/walrus_site/Move.toml @@ -1,6 +1,9 @@ [package] name = "walrus_site" +license = "Apache-2.0" +authors = ["Mysten Labs "] version = "0.0.1" +edition = "2024.beta" [dependencies] Sui = { git = "https://github.com/MystenLabs/sui.git", subdir = "crates/sui-framework/packages/sui-framework", rev = "framework/testnet" } diff --git a/move/walrus_site/sources/site.move b/move/walrus_site/sources/site.move index cc8d167c..705c9e76 100644 --- a/move/walrus_site/sources/site.move +++ b/move/walrus_site/sources/site.move @@ -1,20 +1,23 @@ /// The module exposes the functionality to create and update Walrus sites. module walrus_site::site { - use std::option::Option; - use sui::object::{Self, UID}; - use sui::tx_context::TxContext; use sui::dynamic_field as df; use std::string::String; use sui::vec_map; + /// The name of the dynamic field containing the routes. + const ROUTES_FIELD: vector = b"routes"; + + /// An insertion of route was attempted, but the related resource does not exist. + const EResourceDoesNotExist: u64 = 0; + /// The site published on Sui. - struct Site has key, store { + public struct Site has key, store { id: UID, name: String, } /// A resource in a site. - struct Resource has store, drop { + public struct Resource has store, drop { path: String, // Response, Representation and Payload headers // regarding the contents of the resource. @@ -29,10 +32,15 @@ module walrus_site::site { /// Representation of the resource path. /// /// Ensures there are no namespace collisions in the dynamic fields. - struct ResourcePath has copy, store, drop { + public struct ResourcePath has copy, store, drop { path: String, } + /// The routes for a site. + public struct Routes has store, drop { + route_list: vec_map::VecMap, + } + /// Creates a new site. public fun new_site(name: String, ctx: &mut TxContext): Site { Site { @@ -57,14 +65,10 @@ module walrus_site::site { /// Adds a header to the Resource's headers vector. public fun add_header(resource: &mut Resource, name: String, value: String) { - // Will throw an exception if duplicate key. - vec_map::insert( - &mut resource.headers, - name, - value - ); + resource.headers.insert(name, value); } + /// Creates a new resource path. fun new_path(path: String): ResourcePath { ResourcePath { path } } @@ -96,8 +100,58 @@ module walrus_site::site { /// Changes the path of a resource on a site. public fun move_resource(site: &mut Site, old_path: String, new_path: String) { - let resource = remove_resource(site, old_path); + let mut resource = remove_resource(site, old_path); resource.path = new_path; add_resource(site, resource); } + + // Routes. + + /// Creates a new `Routes` object. + fun new_routes(): Routes { + Routes { route_list: vec_map::empty() } + } + + /// Inserts a route into the `Routes` object. + /// + /// The insertion operation fails if the route already exists. + fun routes_insert(routes: &mut Routes, route: String, resource_path: String) { + routes.route_list.insert(route, resource_path); + } + + /// Removes a route from the `Routes` object. + fun routes_remove(routes: &mut Routes, route: &String): (String, String) { + routes.route_list.remove(route) + } + + // Routes management on the site. + + /// Add the routes dynamic field to the site. + public fun create_routes(site: &mut Site) { + let routes = new_routes(); + df::add(&mut site.id, ROUTES_FIELD, routes); + } + + /// Remove all routes from the site. + public fun remove_all_routes_if_exist(site: &mut Site): Option { + df::remove_if_exists(&mut site.id, ROUTES_FIELD) + } + + /// Add a route to the site. + /// + /// The insertion operation fails: + /// - if the route already exists; or + /// - if the related resource path does not already exist as a dynamic field on the site. + public fun insert_route(site: &mut Site, route: String, resource_path: String) { + let path_obj = new_path(resource_path); + assert!(df::exists_(&site.id, path_obj), EResourceDoesNotExist); + let routes = df::borrow_mut(&mut site.id, ROUTES_FIELD); + routes_insert(routes, route, resource_path); + } + + /// Remove a route from the site. + public fun remove_route(site: &mut Site, route: &String): (String, String) { + let routes = df::borrow_mut(&mut site.id, ROUTES_FIELD); + routes_remove(routes, route) + } } diff --git a/portal/common/lib/bcs_data_parsing.ts b/portal/common/lib/bcs_data_parsing.ts index 2863c6de..0ee97242 100644 --- a/portal/common/lib/bcs_data_parsing.ts +++ b/portal/common/lib/bcs_data_parsing.ts @@ -41,3 +41,7 @@ export function DynamicFieldStruct(K: BcsType, V: BcsType) { value: V, }); } + +export const RoutesStruct = bcs.struct("Routes", { + routes_list: bcs.map(bcs.string(), bcs.string()) +}) diff --git a/portal/common/lib/constants.ts b/portal/common/lib/constants.ts index 956b2905..b360d1b0 100644 --- a/portal/common/lib/constants.ts +++ b/portal/common/lib/constants.ts @@ -1,16 +1,16 @@ // Copyright (c) Mysten Labs, Inc. // SPDX-License-Identifier: Apache-2.0 -export const NETWORK = "testnet" -export const AGGREGATOR = "https://aggregator-devnet.walrus.space:443" -export const SITE_PACKAGE = "0xe15cd956d3f54ad0b6608b01b96e9999d66552dfd025e698ac16cd0df1787a25" -export const MAX_REDIRECT_DEPTH = 3 +export const NETWORK = "testnet"; +export const AGGREGATOR = "https://aggregator-devnet.walrus.space:443"; +export const SITE_PACKAGE = "0xee40a3dddc38e3c9f594f74f85ae3bda75b0ed05c6f2359554126409cf7e8e9d"; +export const MAX_REDIRECT_DEPTH = 3; export const SITE_NAMES: { [key: string]: string } = { // Any hardcoded (non suins) name -> object_id mappings go here // e.g., // landing: "0x1234..." }; // The default portal to redirect to if the browser does not support service workers. -export const FALLBACK_PORTAL = "blob.store" +export const FALLBACK_PORTAL = "blob.store"; // The string representing the ResourcePath struct in the walrus_site package. export const RESOURCE_PATH_MOVE_TYPE = SITE_PACKAGE + "::site::ResourcePath"; diff --git a/portal/common/lib/page_fetching.ts b/portal/common/lib/page_fetching.ts index 37c4b9bc..264e8ede 100644 --- a/portal/common/lib/page_fetching.ts +++ b/portal/common/lib/page_fetching.ts @@ -3,7 +3,7 @@ import { getFullnodeUrl, SuiClient } from "@mysten/sui/client"; import { NETWORK } from "./constants"; -import { DomainDetails, isResource } from "./types/index"; +import { DomainDetails, isResource, isRoutes } from "./types/index"; import { subdomainToObjectId, HEXtoBase36 } from "./objectId_operations"; import { resolveSuiNsAddress, hardcodedSubdmains } from "./suins"; import { fetchResource } from "./resource"; @@ -16,11 +16,16 @@ import { import { aggregatorEndpoint } from "./aggregator"; import { toB64 } from "@mysten/bcs"; import { sha256 } from "./crypto"; +import { getRoutes, matchPathToRoute } from "./routing"; +import { RoutingCacheInterface } from "./routing_cache_interface"; +import { Routes, Empty, isEmpty } from "./types/index"; /** * Resolves the subdomain to an object ID, and gets the corresponding resources. */ -export async function resolveAndFetchPage(parsedUrl: DomainDetails): Promise { +export async function resolveAndFetchPage( + parsedUrl: DomainDetails, cache?: RoutingCacheInterface +): Promise { const rpcUrl = getFullnodeUrl(NETWORK); const client = new SuiClient({ url: rpcUrl }); const resolveObjectResult = await resolveObjectId(parsedUrl, client); @@ -28,7 +33,38 @@ export async function resolveAndFetchPage(parsedUrl: DomainDetails): Promise { + // TODO: On the portal side, fetch the route object, store it to a routes object. + // Then pass the routes object to the fetchPage params. const result = await fetchResource(client, objectId, path, new Set()); if (!isResource(result) || !result.blob_id) { if (path !== "/404.html") { diff --git a/portal/common/lib/routing.test.ts b/portal/common/lib/routing.test.ts new file mode 100644 index 00000000..71669465 --- /dev/null +++ b/portal/common/lib/routing.test.ts @@ -0,0 +1,42 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +import { getRoutes, matchPathToRoute } from "./routing"; +import { test, expect } from 'vitest'; +import { SuiClient, getFullnodeUrl } from "@mysten/sui/client"; +import { NETWORK } from "./constants"; + +const snakeSiteObjectId = '0x3e01b1b8bf0e54f7843596345faff146f1047e304410ed2eb85d5f67ad404206'; +test.skip('getRoutes', async () => { + // TODO: when you make sure get_routes fetches + // the Routes dynamic field, mock the request. + const rpcUrl = getFullnodeUrl(NETWORK); + const client = new SuiClient({ url: rpcUrl }); + const routes = await getRoutes(client, snakeSiteObjectId); + console.log(routes) +}); + +const routesExample = { + routes_list: new Map([ + ['/*', '/default.html'], + ['/somewhere/else', 'else.jpeg'], + ['/somewhere/else/*', 'star-else.gif'], + ['/path/to/*', '/somewhere.html'], + ]) +}; + +const testCases = [ + ["/path/to/somewhere/", "/somewhere.html"], + ["/somewhere/else", 'else.jpeg'], + ["/", "/default.html"], + ["/somewhere", "/default.html"], + ["/somewhere/else/star", "star-else.gif"], + ["/somewhere/else/", 'star-else.gif'], +] + +testCases.forEach(([requestPath, expected]) => { + test(`matchPathToRoute: "${requestPath}" -> "${expected}"`, () => { + const match = matchPathToRoute(requestPath, routesExample) + expect(match).toEqual(expected) + }) +}); diff --git a/portal/common/lib/routing.ts b/portal/common/lib/routing.ts new file mode 100644 index 00000000..30036359 --- /dev/null +++ b/portal/common/lib/routing.ts @@ -0,0 +1,101 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +import { getFullnodeUrl, SuiClient, SuiObjectResponse } from "@mysten/sui/client"; +import { Routes, } from "./types"; +import { + DynamicFieldStruct, + RoutesStruct, +} from "./bcs_data_parsing"; +import { bcs, fromB64 } from "@mysten/bcs"; + + +/** + * Gets the Routes dynamic field of the site object. + * Returns the extracted routes_list map to use for future requests, + * and redirects the paths matched accordingly. + * + * @param siteObjectId - The ID of the site object. + * @returns The routes list. + */ +export async function getRoutes( + client: SuiClient, siteObjectId: string +): Promise { + const routesDF = await fetchRoutesDynamicField(client, siteObjectId); + if (!routesDF.data) { + console.warn("No routes dynamic field found for site object."); + return; + } + const routesObj = await fetchRoutesObject(client, routesDF.data.objectId); + const objectData = routesObj.data; + if (objectData && objectData.bcs && objectData.bcs.dataType === "moveObject") { + return parseRoutesData(objectData.bcs.bcsBytes); + } + throw new Error("Routes object data could not be fetched."); +} + +/** + * Fetches the dynamic field object for routes. + * + * @param client - The SuiClient instance. + * @param siteObjectId - The ID of the site object. + * @returns The dynamic field object for routes. + */ +async function fetchRoutesDynamicField( + client: SuiClient, siteObjectId: string +): Promise { + return await client.getDynamicFieldObject({ + parentId: siteObjectId, + name: { type: "vector", value: "routes" }, + }); +} + +/** + * Fetches the routes object using the dynamic field object ID. + * + * @param client - The SuiClient instance. + * @param objectId - The ID of the dynamic field object. + * @returns The routes object. + */ +async function fetchRoutesObject(client: SuiClient, objectId: string): Promise { + return await client.getObject({ + id: objectId, + options: { showBcs: true } + }); +} + +/** + * Parses the routes data from the BCS bytes. + * + * @param bcsBytes - The BCS bytes of the routes object. + * @returns The parsed routes data. + */ +function parseRoutesData(bcsBytes: string): Routes { + const df = DynamicFieldStruct( + // BCS declaration of the ROUTES_FIELD in site.move. + bcs.vector(bcs.u8()), + // The value of the df, i.e. the Routes Struct. + RoutesStruct + ).parse(fromB64(bcsBytes)); + + return df.value as any as Routes; +} + +/** + * Matches the path to the appropriate route. + * Path patterns in the routes list are sorted by length in descending order. + * Then the first match is returned. + * + * @param path - The path to match. + * @param routes - The routes to match against. + */ +export function matchPathToRoute(path: string, routes: Routes): string | undefined { + // TODO: improve this using radix trees. + const sortedRoutes: Array<[string, string]> = Array.from( + routes.routes_list.entries() + ).sort((current, next) => next[0].length - current[0].length); + const matchedRoute = sortedRoutes.find( + ([pattern, _]) => new RegExp(`^${pattern.replace('*', '.*')}$`).test(path) + ); + return matchedRoute? matchedRoute[1] : undefined +} diff --git a/portal/common/lib/routing_cache_interface.ts b/portal/common/lib/routing_cache_interface.ts new file mode 100644 index 00000000..b47d166f --- /dev/null +++ b/portal/common/lib/routing_cache_interface.ts @@ -0,0 +1,19 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +import { Routes, Empty } from "./types"; + +/** + * Abstract class for a cache that stores Routes objects, + * to be able to swap out different caches between + * different portal implementations. + */ +export interface RoutingCacheInterface { + get(key: string): Promise; + /// A note on Empty: + /// This is used to indicate that the cache does not have a value for the key. + /// It is used to differentiate between a cache miss and a cache hit with an empty value. + /// Cache hit with empty values should not trigger a fetch from the fullnode. + set(key: string, value: Routes | Empty): Promise; + delete(key: string): Promise; +} diff --git a/portal/common/lib/types/index.ts b/portal/common/lib/types/index.ts index e9fbf391..e38f27e9 100644 --- a/portal/common/lib/types/index.ts +++ b/portal/common/lib/types/index.ts @@ -54,3 +54,33 @@ export function isVersionedResource(resource: any): resource is VersionedResourc && 'version' in resource && 'objectId' in resource; } + +/** + * Routes is an opitonal dynamic field object belonging to each site. + */ +export type Routes = { + routes_list: Map; +} + +/** + * Type guard for the Routes type. + */ +export function isRoutes(obj: any): obj is Routes { + return ( + obj && + typeof obj.routes_list === 'object' && + obj.routes_list instanceof Map + ); +} + +/// An empty object that cannot have any properties. +export type Empty = Record; + +/** + * Checks if an object is of type Empty. + * @param obj The object to check. + * @returns True if the object is Empty, false otherwise. + */ +export function isEmpty(obj: any): obj is Empty { + return Object.keys(obj).length === 0; +} diff --git a/portal/server/next-env.d.ts b/portal/server/next-env.d.ts index 8e8ff34b..d5d7d918 100644 --- a/portal/server/next-env.d.ts +++ b/portal/server/next-env.d.ts @@ -5,4 +5,3 @@ /// // NOTE: This file should not be edited -// see https://nextjs.org/docs/basic-features/typescript for more information. diff --git a/portal/worker/src/caching.ts b/portal/worker/src/caching.ts index 628486f5..9a6ab5c8 100644 --- a/portal/worker/src/caching.ts +++ b/portal/worker/src/caching.ts @@ -5,6 +5,7 @@ import { resolveAndFetchPage } from "@lib/page_fetching"; import { SuiClient, getFullnodeUrl } from "@mysten/sui/client"; import { NETWORK } from "@lib/constants"; import { DomainDetails } from "@lib/types"; +import { RoutingCacheInterface } from "@lib/routing_cache_interface"; const CACHE_NAME = "walrus-sites-cache"; const CACHE_EXPIRATION_TIME = 24 * 60 * 60 * 1000 // 24 hours in milliseconds @@ -13,7 +14,7 @@ const CACHE_EXPIRATION_TIME = 24 * 60 * 60 * 1000 // 24 hours in milliseconds * Respond to the request using the cache API. */ export default async function resolveWithCache( - parsedUrl: DomainDetails, urlString: string + parsedUrl: DomainDetails, urlString: string, routingCache: RoutingCacheInterface ): Promise { const cache = await caches.open(CACHE_NAME); const cachedResponse = await cache.match(urlString); @@ -30,7 +31,7 @@ export default async function resolveWithCache( } } console.log("Cache miss!", urlString); - const resolvedPage = await resolveAndFetchPage(parsedUrl); + const resolvedPage = await resolveAndFetchPage(parsedUrl, routingCache); cache.put(urlString, resolvedPage.clone()); return resolvedPage; diff --git a/portal/worker/src/routing_cache.ts b/portal/worker/src/routing_cache.ts new file mode 100644 index 00000000..443fa110 --- /dev/null +++ b/portal/worker/src/routing_cache.ts @@ -0,0 +1,96 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +import { RoutingCacheInterface } from "@lib/routing_cache_interface"; +import { Routes, isRoutes, Empty, isEmpty } from "@lib/types"; + +/** + * ServiceWorkerRoutingCache is a class that implements the RoutingCacheInterface. + * It provides methods to initialize a cache, get, set, and delete routing cache entries. + * The cache is used to store routing information for service workers. + */ +export class ServiceWorkerRoutingCache implements RoutingCacheInterface { + /// The IndexedDB database used to store the cache. + private cache: IDBDatabase; + /// Used to ensure that the cache is initialized before any operations are performed. + private initialized?: Promise; + + /// Initializes the routing cache. Using this instead of a + /// constructor because the constructor cannot be async. + async init(window: ServiceWorkerGlobalScope): Promise { + // If the cache is already initialized, return. + if (this.initialized != undefined) return; + + console.log("Initializing routing cache using the IndexedDB API."); + this.initialized = new Promise((resolve, reject) => { + const request = window.indexedDB.open("routing-cache", 1); + request.onupgradeneeded = (_event) => { + const db = request.result; + const objectStore = db.createObjectStore('routes'); + objectStore.createIndex('siteObjectId', 'siteObjectId', { unique: true }); + }; + request.onsuccess = () => { + this.cache = request.result; + resolve(true); + }; + request.onerror = () => { + reject(request.error); + }; + }); + } + + async get(key: string): Promise { + await this.initialized; // Make sure the cache is initialized before using it. + return new Promise((resolve, reject) => { + const transaction = this.cache.transaction("routes", "readonly"); + const store = transaction.objectStore("routes"); + const request = store.get(key); + request.onsuccess = () => { + const value = request.result; + if (!value) { + console.log("Routing cache miss: ", value); + resolve(undefined); + } + if (isRoutes(value) || isEmpty(value)) { + console.log(`Routing cache hit for ${key}: `, value); + resolve(value); + } + resolve(undefined); + }; + request.onerror = () => { + reject(request.error); + }; + }); + } + + async set(key: string, value: Routes | Empty): Promise { + await this.initialized; // Make sure the cache is initialized before using it. + return new Promise((resolve, reject) => { + console.log("Setting routing cache. KV Pair: ", key, value) + const transaction = this.cache.transaction("routes", "readwrite"); + const store = transaction.objectStore("routes"); + const request = store.put(value, key); + request.onsuccess = () => { + resolve(); + }; + request.onerror = () => { + reject(request.error); + }; + }); + } + + async delete(key: string): Promise { + await this.initialized; // Make sure the cache is initialized before using it. + return new Promise((resolve, reject) => { + const transaction = this.cache.transaction("routes", "readwrite"); + const store = transaction.objectStore("routes"); + const request = store.delete(key); + request.onsuccess = () => { + resolve(); + }; + request.onerror = () => { + reject(request.error); + }; + }); + } +} diff --git a/portal/worker/src/walrus-sites-portal-register-sw.ts b/portal/worker/src/walrus-sites-portal-register-sw.ts index bd188816..b7d690ac 100644 --- a/portal/worker/src/walrus-sites-portal-register-sw.ts +++ b/portal/worker/src/walrus-sites-portal-register-sw.ts @@ -21,33 +21,33 @@ function main() { } else if (reg.active) { console.log("SW active, error?"); // Previously-installed SW should have redirected this request to different page - handleError(new Error("Service Worker is installed but not redirecting")); + handleError(); } }) .catch(handleError); } else { const currentUrl = new URL(window.location.href); - console.warn("This browser does not yet support Walrus Sites 💔, redirecting to blob.store"); - const domainDetails = getSubdomainAndPath(currentUrl) + console.warn( + "This browser does not yet support Walrus Sites 💔, redirecting to blob.store", + ); + const domainDetails = getSubdomainAndPath(currentUrl); window.location.href = new URL( `${currentUrl.pathname}${currentUrl.search}${currentUrl.hash}`, `https://${ - domainDetails.subdomain ? - domainDetails.subdomain + '.' - : '' - }${FALLBACK_PORTAL}` + domainDetails.subdomain ? domainDetails.subdomain + "." : "" + }${FALLBACK_PORTAL}`, ).toString(); } } -function handleError(error) { +function handleError() { displayErrorMessage(swNotLoadingNode()); } function swNotLoadingNode() { return titleSubtitleNode( "Oh! Something's not right 🚧", - "Please try refreshing the page or unregistering the service worker." + "Please try refreshing the page or unregistering the service worker.", ); } diff --git a/portal/worker/src/walrus-sites-sw.ts b/portal/worker/src/walrus-sites-sw.ts index 57e8a4fe..653c4f63 100644 --- a/portal/worker/src/walrus-sites-sw.ts +++ b/portal/worker/src/walrus-sites-sw.ts @@ -6,6 +6,7 @@ import { redirectToAggregatorUrlResponse, redirectToPortalURLResponse } from "@l import { getBlobIdLink, getObjectIdLink } from "@lib/links"; import resolveWithCache from "./caching"; import { resolveAndFetchPage } from "@lib/page_fetching"; +import { ServiceWorkerRoutingCache } from "./routing_cache"; // This is to get TypeScript to recognize `clients` and `self` Default type of `self` is // `WorkerGlobalScope & typeof globalThis` https://github.com/microsoft/TypeScript/issues/14877 @@ -55,7 +56,11 @@ self.addEventListener("fetch", async (event) => { console.warn("Cache API not available"); response = resolveAndFetchPage(parsedUrl); } else { - response = resolveWithCache(parsedUrl, urlString); + const swRoutingCache = new ServiceWorkerRoutingCache(); + // window is not available in SW. Use self instead + // https://stackoverflow.com/a/11237259 + swRoutingCache.init(self); + response = resolveWithCache(parsedUrl, urlString, swRoutingCache); } event.respondWith(response); return; diff --git a/site-builder/assets/builder-example.yaml b/site-builder/assets/builder-example.yaml index 91af00b3..aae572f7 100644 --- a/site-builder/assets/builder-example.yaml +++ b/site-builder/assets/builder-example.yaml @@ -1,6 +1,6 @@ # module: site # portal: walrus.site -package: 0xe15cd956d3f54ad0b6608b01b96e9999d66552dfd025e698ac16cd0df1787a25 +package: 0xee40a3dddc38e3c9f594f74f85ae3bda75b0ed05c6f2359554126409cf7e8e9d # general: # rpc_url: https://fullnode.testnet.sui.io:443 # wallet: /path/to/.sui/sui_config/client.yaml diff --git a/site-builder/src/main.rs b/site-builder/src/main.rs index ad9e8e04..d21407ac 100644 --- a/site-builder/src/main.rs +++ b/site-builder/src/main.rs @@ -5,6 +5,7 @@ mod display; mod preprocessor; mod publish; mod site; +mod summary; mod util; mod walrus; use std::path::PathBuf; @@ -14,12 +15,12 @@ use clap::{Parser, Subcommand}; use futures::TryFutureExt; use publish::{ContinuousEditing, PublishOptions, SiteEditor, WhenWalrusUpload}; use serde::Deserialize; -use site::manager::SiteIdentifier; +use site::{manager::SiteIdentifier, RemoteSiteFactory}; use sui_types::base_types::ObjectID; use crate::{ preprocessor::Preprocessor, - util::{get_existing_resource_ids, id_to_base36, load_wallet_context}, + util::{id_to_base36, load_wallet_context}, }; // Define the `GIT_REVISION` and `VERSION` consts. @@ -258,8 +259,9 @@ async fn run() -> Result<()> { // below will be monitored for changes. Commands::Sitemap { object } => { let wallet = load_wallet_context(&config.general.wallet)?; - let all_dynamic_fields = - get_existing_resource_ids(&wallet.get_client().await?, object).await?; + let all_dynamic_fields = RemoteSiteFactory::new(&wallet.get_client().await?, object) + .get_existing_resources() + .await?; println!("Pages in site at object id: {}", object); for (name, id) in all_dynamic_fields { println!(" - {:<40} {:?}", name, id); diff --git a/site-builder/src/publish.rs b/site-builder/src/publish.rs index 407e6347..9374da31 100644 --- a/site-builder/src/publish.rs +++ b/site-builder/src/publish.rs @@ -22,8 +22,9 @@ use crate::{ site::{ config::WSResources, manager::{SiteIdentifier, SiteManager}, - resource::{OperationsSummary, ResourceManager}, + resource::ResourceManager, }, + summary::{SiteDataDiffSummary, Summarizable}, util::{ get_site_id_from_response, id_to_base36, @@ -145,7 +146,7 @@ impl SiteEditor { async fn run_single_edit( &self, - ) -> Result<(SuiAddress, SuiTransactionBlockResponse, OperationsSummary)> { + ) -> Result<(SuiAddress, SuiTransactionBlockResponse, SiteDataDiffSummary)> { if self.publish_options.list_directory { display::action(format!("Preprocessing: {}", self.directory().display())); Preprocessor::preprocess(self.directory())?; @@ -177,9 +178,9 @@ impl SiteEditor { "Parsing the directory {} and locally computing blob IDs", self.directory().to_string_lossy() )); - resource_manager.read_dir(self.directory())?; + let local_site_data = resource_manager.read_dir(self.directory())?; display::done(); - tracing::debug!(resources=%resource_manager.resources, "resources loaded from directory"); + tracing::debug!(?local_site_data, "resources loaded from directory"); let site_manager = SiteManager::new( self.config.clone(), @@ -190,7 +191,7 @@ impl SiteEditor { self.when_upload.clone(), ) .await?; - let (response, summary) = site_manager.update_site(&resource_manager).await?; + let (response, summary) = site_manager.update_site(&local_site_data).await?; Ok((site_manager.active_address()?, response, summary)) } @@ -233,7 +234,7 @@ fn print_summary( address: &SuiAddress, site_id: &SiteIdentifier, response: &SuiTransactionBlockResponse, - summary: &OperationsSummary, + summary: &impl Summarizable, ) -> Result<()> { if let Some(SuiTransactionBlockEffects::V1(eff)) = response.effects.as_ref() { if let SuiExecutionStatus::Failure { error } = &eff.status { @@ -245,7 +246,7 @@ fn print_summary( } display::header("Execution completed"); - println!("{}\n", summary); + println!("{}\n", summary.to_summary()); let object_id = match site_id { SiteIdentifier::ExistingSite(id) => { println!("Site object ID: {}", id); diff --git a/site-builder/src/site.rs b/site-builder/src/site.rs index bf61588b..706ff198 100644 --- a/site-builder/src/site.rs +++ b/site-builder/src/site.rs @@ -8,3 +8,288 @@ pub mod macros; pub mod config; pub mod manager; pub mod resource; + +use std::{ + collections::{BTreeMap, HashMap}, + str::FromStr, +}; + +use anyhow::{anyhow, Result}; +use resource::{MapWrapper, ResourceInfo, ResourceOp, ResourceSet}; +use serde::{Deserialize, Serialize}; +use sui_sdk::{ + rpc_types::{SuiMoveStruct, SuiMoveValue, SuiObjectDataOptions}, + SuiClient, +}; +use sui_types::{ + base_types::ObjectID, + dynamic_field::{DynamicFieldInfo, DynamicFieldName}, + TypeTag, +}; + +use crate::util::{get_struct_from_object_response, handle_pagination}; + +pub const SITE_MODULE: &str = "site"; + +/// The routes of a site. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Routes(pub BTreeMap); + +impl Routes { + pub fn empty() -> Self { + Routes(BTreeMap::new()) + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + /// Checks if the routes are different. + pub fn diff(&self, start: &Self) -> RouteOps { + tracing::debug!(?self, ?start, "diffing routes INNER"); + if self.0 == start.0 { + RouteOps::Unchanged + } else { + RouteOps::Replace(self.clone()) + } + } +} + +impl TryFrom for Routes { + type Error = anyhow::Error; + + fn try_from(source: SuiMoveStruct) -> Result { + let routes: MapWrapper = + get_dynamic_field!(source, "route_list", SuiMoveValue::Struct)?.try_into()?; + Ok(Self(routes.0)) + } +} + +#[derive(Debug, Clone)] +pub enum RouteOps { + Unchanged, + Replace(Routes), +} + +impl RouteOps { + pub fn is_unchanged(&self) -> bool { + matches!(self, RouteOps::Unchanged) + } +} + +/// The diff between two site data. +#[derive(Debug)] +pub struct SiteDataDiff<'a> { + /// The operations to perform on the resources. + pub resource_ops: Vec>, + pub route_ops: RouteOps, +} + +impl SiteDataDiff<'_> { + /// Returns `true` if there are updates to be made. + pub fn has_updates(&self) -> bool { + !self.resource_ops.is_empty() || !self.route_ops.is_unchanged() + } +} + +/// The site on chain. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SiteData { + resources: ResourceSet, + routes: Option, +} + +impl SiteData { + pub fn new(resources: ResourceSet, routes: Option) -> Self { + Self { resources, routes } + } + + pub fn empty() -> Self { + Self { + resources: ResourceSet::empty(), + routes: None, + } + } + + // TODO(giac): rename start and reorder the direction of the diff. + /// Returns the operations to perform to transform the start set into self. + pub fn diff<'a>(&'a self, start: &'a SiteData) -> SiteDataDiff<'a> { + SiteDataDiff { + resource_ops: self.resources.diff(&start.resources), + route_ops: self.routes_diff(start), + } + } + + /// Returns the operations to perform to replace all resources in self with the ones in other. + pub fn replace_all<'a>(&'a self, other: &'a SiteData) -> SiteDataDiff<'a> { + SiteDataDiff { + resource_ops: self.resources.replace_all(&other.resources), + route_ops: self.routes_diff(other), + } + } + + fn routes_diff(&self, start: &Self) -> RouteOps { + tracing::debug!(?self.routes, ?start.routes, "diffing routes"); + match (&self.routes, &start.routes) { + (Some(r), Some(s)) => r.diff(s), + (None, Some(_)) => RouteOps::Replace(Routes::empty()), + (Some(s), None) => RouteOps::Replace(s.clone()), + _ => RouteOps::Unchanged, + } + } +} + +/// Fetches remote sites. +pub struct RemoteSiteFactory<'a> { + sui_client: &'a SuiClient, + package_id: ObjectID, +} + +impl RemoteSiteFactory<'_> { + /// Creates a new remote site factory. + pub fn new(sui_client: &SuiClient, package_id: ObjectID) -> RemoteSiteFactory { + RemoteSiteFactory { + sui_client, + package_id, + } + } + + /// Gets the remote site representation stored on chain + pub async fn get_from_chain(&self, site_id: ObjectID) -> Result { + let dynamic_fields = self.get_all_dynamic_field_info(site_id).await?; + let resource_ids = self.resources_from_dynamic_fields(&dynamic_fields)?; + let resources = futures::future::try_join_all( + resource_ids + .into_values() + .map(|id| self.get_remote_resource_info(id)), + ) + .await?; + let routes = self.get_routes(site_id).await?; + + Ok(SiteData { + resources: ResourceSet::from_iter(resources), + routes, + }) + } + + async fn get_routes(&self, site_id: ObjectID) -> Result> { + let response = self + .sui_client + .read_api() + .get_dynamic_field_object( + site_id, + DynamicFieldName { + type_: TypeTag::Vector(Box::new(TypeTag::U8)), + value: "routes".into(), + }, + ) + .await?; + let dynamic_field = get_struct_from_object_response(&response)?; + let routes = + get_dynamic_field!(dynamic_field, "value", SuiMoveValue::Struct)?.try_into()?; + Ok(Some(routes)) + } + + /// Gets all the resources and their object ids from chain. + pub async fn get_existing_resources(&self) -> Result> { + let dynamic_fields = self.get_all_dynamic_field_info(self.package_id).await?; + self.resources_from_dynamic_fields(&dynamic_fields) + } + + async fn get_all_dynamic_field_info( + &self, + object_id: ObjectID, + ) -> Result> { + let iter = handle_pagination(|cursor| { + self.sui_client + .read_api() + .get_dynamic_fields(object_id, cursor, None) + }) + .await? + .collect(); + Ok(iter) + } + + /// Get the resource that is hosted on chain at the given object ID. + async fn get_remote_resource_info(&self, object_id: ObjectID) -> Result { + let object = get_struct_from_object_response( + &self + .sui_client + .read_api() + .get_object_with_options(object_id, SuiObjectDataOptions::new().with_content()) + .await?, + )?; + get_dynamic_field!(object, "value", SuiMoveValue::Struct)?.try_into() + } + + /// Filters the dynamic fields to get the resource object IDs. + fn resources_from_dynamic_fields( + &self, + dynamic_fields: &[DynamicFieldInfo], + ) -> Result> { + let type_tag = self.resource_path_tag(); + Ok(dynamic_fields + .iter() + .filter_map(|field| { + self.get_path_from_info(field, &type_tag) + .map(|path| (path, field.object_id)) + }) + .collect::>()) + } + + /// Gets the path of the resource from the dynamic field. + fn get_path_from_info(&self, field: &DynamicFieldInfo, name_tag: &TypeTag) -> Option { + if field.name.type_ != *name_tag { + return None; + } + field + .name + .value + .as_object() + .and_then(|obj| obj.get("path")) + .and_then(|p| p.as_str()) + .map(|s| s.to_owned()) + } + + /// Gets the type tag for the ResourcePath move struct + fn resource_path_tag(&self) -> TypeTag { + TypeTag::from_str(&format!("{}::{SITE_MODULE}::ResourcePath", self.package_id)) + .expect("this is a valid type tag construction") + } +} + +#[cfg(test)] +mod tests { + use super::{Routes, SiteData}; + use crate::site::resource::ResourceSet; + + fn routes_from_pair(key: &str, value: &str) -> Option { + Some(Routes( + vec![(key.to_owned(), value.to_owned())] + .into_iter() + .collect(), + )) + } + + #[test] + fn test_routes_diff() { + let cases = vec![ + (None, None, false), + (routes_from_pair("a", "b"), None, true), + (None, routes_from_pair("a", "b"), true), + ( + routes_from_pair("a", "b"), + routes_from_pair("a", "b"), + false, + ), + (routes_from_pair("a", "a"), routes_from_pair("a", "b"), true), + ]; + + for (this_routes, other_routes, has_updates) in cases { + let this = SiteData::new(ResourceSet::empty(), this_routes); + let other = SiteData::new(ResourceSet::empty(), other_routes); + let diff = this.diff(&other); + assert_eq!(diff.has_updates(), has_updates); + } + } +} diff --git a/site-builder/src/site/builder.rs b/site-builder/src/site/builder.rs index f07afb40..5efa8825 100644 --- a/site-builder/src/site/builder.rs +++ b/site-builder/src/site/builder.rs @@ -11,7 +11,10 @@ use sui_types::{ TypeTag, }; -use super::resource::{Resource, ResourceOp}; +use super::{ + resource::{Resource, ResourceOp}, + RouteOps, +}; pub struct SitePtb { pt_builder: ProgrammableTransactionBuilder, @@ -93,7 +96,7 @@ impl SitePtb { } impl SitePtb { - pub fn add_operations<'a>( + pub fn add_resource_operations<'a>( &mut self, calls: impl IntoIterator>, ) -> Result<()> { @@ -106,6 +109,20 @@ impl SitePtb { Ok(()) } + /// Adds move calls to update the routes on the object. + pub fn add_route_operations(&mut self, route_ops: &RouteOps) -> Result<()> { + if let RouteOps::Replace(new_routes) = route_ops { + self.remove_routes()?; + if !new_routes.is_empty() { + self.create_routes()?; + for (name, value) in new_routes.0.iter() { + self.add_route(name, value)?; + } + } + } + Ok(()) + } + pub fn transfer_site(&mut self, recipient: SuiAddress) { self.transfer_arg(recipient, self.site_argument); } @@ -160,15 +177,54 @@ impl SitePtb { /// Adds the header to the given resource argument. fn add_header(&mut self, resource_arg: Argument, name: &str, value: &str) -> Result<()> { - let name_input = self.pt_builder.input(pure_call_arg(&name.to_owned())?)?; + self.add_key_value_to_argument("add_header", resource_arg, name, value) + } + + /// Adds the move calls to add key and value to the argument. + fn add_key_value_to_argument( + &mut self, + fn_name: &str, + arg: Argument, + key: &str, + value: &str, + ) -> Result<()> { + let name_input = self.pt_builder.input(pure_call_arg(&key.to_owned())?)?; let value_input = self.pt_builder.input(pure_call_arg(&value.to_owned())?)?; self.add_programmable_move_call( - Identifier::new("add_header")?, + Identifier::new(fn_name)?, vec![], - vec![resource_arg, name_input, value_input], + vec![arg, name_input, value_input], ); Ok(()) } + + // Routes + + /// Adds the move calls to create a new routes object. + fn create_routes(&mut self) -> Result<()> { + self.add_programmable_move_call( + Identifier::new("create_routes")?, + vec![], + vec![self.site_argument], + ); + Ok(()) + } + + /// Adds the move calls to remove the routes object. + fn remove_routes(&mut self) -> Result<()> { + self.add_programmable_move_call( + Identifier::new("remove_all_routes_if_exist")?, + vec![], + vec![self.site_argument], + ); + Ok(()) + } + + /// Adds the move calls add a route to the routes object. + fn add_route(&mut self, name: &str, value: &str) -> Result<()> { + tracing::debug!(name=%name, value=%value, "new Move call: adding route"); + self.add_key_value_to_argument("insert_route", self.site_argument, name, value) + } } pub fn pure_call_arg(arg: &T) -> Result { diff --git a/site-builder/src/site/config.rs b/site-builder/src/site/config.rs index 51d095ce..e0b11cb2 100644 --- a/site-builder/src/site/config.rs +++ b/site-builder/src/site/config.rs @@ -4,17 +4,19 @@ use std::{collections::BTreeMap, path::Path}; use anyhow::{Context, Result}; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; -use super::resource::HttpHeaders; +use super::{resource::HttpHeaders, Routes}; /// Deserialized object of the file's `ws-resource.json` contents. -#[derive(Deserialize, Debug)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct WSResources { /// The HTTP headers to be set for the resources. #[serde(skip_serializing_if = "Option::is_none")] pub headers: Option>, - // TODO: "routes"" for client-side routing. + /// The HTTP headers to be set for the resources. + #[serde(skip_serializing_if = "Option::is_none")] + pub routes: Option, } impl WSResources { @@ -35,10 +37,7 @@ mod tests { use super::*; - #[test] - fn test_read_ws_resources() { - let data = r#" - { + const HEADER_DATA: &str = r#" "headers": { "/index.html": { "Content-Type": "application/json", @@ -46,8 +45,21 @@ mod tests { "Cache-Control": "no-cache" } } - } "#; - serde_json::from_str::(data).expect("parsing should succeed"); + + const ROUTE_DATA: &str = r#" + "routes": { + "/*": "/index.html" + } + "#; + + #[test] + fn test_read_ws_resources() { + let header_data = format!("{{{}}}", HEADER_DATA); + serde_json::from_str::(&header_data).expect("parsing should succeed"); + let route_data = format!("{{{}}}", ROUTE_DATA); + serde_json::from_str::(&route_data).expect("parsing should succeed"); + let route_header_data = format!("{{{},{}}}", HEADER_DATA, ROUTE_DATA); + serde_json::from_str::(&route_header_data).expect("parsing should succeed"); } } diff --git a/site-builder/src/site/manager.rs b/site-builder/src/site/manager.rs index c0c28de1..ffa55867 100644 --- a/site-builder/src/site/manager.rs +++ b/site-builder/src/site/manager.rs @@ -5,29 +5,29 @@ use std::{collections::BTreeSet, str::FromStr}; use anyhow::{anyhow, Result}; use sui_keys::keystore::AccountKeystore; -use sui_sdk::{ - rpc_types::{SuiMoveValue, SuiObjectDataOptions, SuiTransactionBlockResponse}, - wallet_context::WalletContext, - SuiClient, -}; +use sui_sdk::{rpc_types::SuiTransactionBlockResponse, wallet_context::WalletContext, SuiClient}; use sui_types::{ base_types::{ObjectID, ObjectRef, SuiAddress}, - transaction::{Argument, ProgrammableTransaction, TransactionData}, + transaction::{ProgrammableTransaction, TransactionData}, Identifier, }; -use super::resource::{OperationsSummary, ResourceInfo, ResourceManager, ResourceOp, ResourceSet}; +use super::{ + builder::SitePtb, + resource::ResourceOp, + RemoteSiteFactory, + SiteData, + SiteDataDiff, + SITE_MODULE, +}; use crate::{ display, publish::WhenWalrusUpload, - site::builder::SitePtb, - util::{self, get_struct_from_object_response}, + summary::SiteDataDiffSummary, walrus::Walrus, Config, }; -const SITE_MODULE: &str = "site"; - /// The identifier for the new or existing site. /// /// Either object ID (existing site) or name (new site). @@ -72,47 +72,33 @@ impl SiteManager { /// or created are published to Walrus. pub async fn update_site( &self, - resource_manager: &ResourceManager, - ) -> Result<(SuiTransactionBlockResponse, OperationsSummary)> { - let ptb = SitePtb::new( - self.config.package, - Identifier::from_str(SITE_MODULE).expect("the str provided is valid"), - )?; - let (ptb, existing_resources, needs_transfer) = match &self.site_id { - SiteIdentifier::ExistingSite(site_id) => ( - ptb.with_call_arg(&self.wallet.get_object_ref(*site_id).await?.into())?, - self.get_existing_resources(*site_id).await?, - false, - ), - SiteIdentifier::NewSite(site_name) => ( - ptb.with_create_site(site_name)?, - ResourceSet::default(), - true, - ), + local_site_data: &SiteData, + ) -> Result<(SuiTransactionBlockResponse, SiteDataDiffSummary)> { + let existing_site = match &self.site_id { + SiteIdentifier::ExistingSite(site_id) => { + RemoteSiteFactory::new(&self.sui_client().await?, self.config.package) + .get_from_chain(*site_id) + .await? + } + SiteIdentifier::NewSite(_) => SiteData::empty(), }; - tracing::debug!(?existing_resources, "checked existing resources"); - let update_operations = if self.when_upload.is_always() { - existing_resources.replace_all(&resource_manager.resources) + tracing::debug!(?existing_site, "checked existing site"); + + let site_updates = if self.when_upload.is_always() { + existing_site.replace_all(local_site_data) } else { - resource_manager.resources.diff(&existing_resources) + local_site_data.diff(&existing_site) }; - tracing::debug!(operations=?update_operations, "list of operations computed"); - - self.publish_to_walrus(&update_operations).await?; + tracing::debug!(operations=?site_updates, "list of operations computed"); - if !update_operations.is_empty() { + if site_updates.has_updates() { + self.publish_to_walrus(&site_updates.resource_ops).await?; display::action("Updating the Walrus Site object on Sui"); - let result = self - .execute_updates(ptb, &update_operations, needs_transfer) - .await?; + let result = self.execute_sui_updates(&site_updates).await?; display::done(); - return Ok((result, update_operations.into())); + return Ok((result, site_updates.into())); } - // TODO(giac) improve this return - Ok(( - SuiTransactionBlockResponse::default(), - update_operations.into(), - )) + Ok((SuiTransactionBlockResponse::default(), site_updates.into())) } /// Publishes the resources to Walrus. @@ -145,19 +131,32 @@ impl SiteManager { Ok(()) } - async fn execute_updates<'b>( + /// Executes the updates on Sui. + async fn execute_sui_updates<'b>( &self, - mut ptb: SitePtb, - updates: &[ResourceOp<'b>], - transfer: bool, + updates: &SiteDataDiff<'b>, ) -> Result { tracing::debug!( address=?self.active_address()?, "starting to update site resources on chain", ); + let ptb = SitePtb::new( + self.config.package, + Identifier::from_str(SITE_MODULE).expect("the str provided is valid"), + )?; - ptb.add_operations(updates)?; - if transfer { + // Add the call arg if we are updating a site, or add the command to create a new site. + let mut ptb = match &self.site_id { + SiteIdentifier::ExistingSite(site_id) => { + ptb.with_call_arg(&self.wallet.get_object_ref(*site_id).await?.into())? + } + SiteIdentifier::NewSite(site_name) => ptb.with_create_site(site_name)?, + }; + + ptb.add_resource_operations(&updates.resource_ops)?; + ptb.add_route_operations(&updates.route_ops)?; + + if self.needs_transfer() { ptb.transfer_site(self.active_address()?); } @@ -182,40 +181,6 @@ impl SiteManager { self.wallet.execute_transaction_may_fail(transaction).await } - async fn get_existing_resources(&self, site_id: ObjectID) -> Result { - let resource_ids = self.get_existing_resource_ids(site_id).await?; - let resources = futures::future::try_join_all( - resource_ids - .into_iter() - .map(|id| self.get_remote_resource_info(id)), - ) - .await?; - Ok(ResourceSet::from_iter(resources)) - } - - /// Get the resources already published to the site. - async fn get_existing_resource_ids(&self, site_id: ObjectID) -> Result> { - Ok( - util::get_existing_resource_ids(&self.sui_client().await?, site_id) - .await? - .into_values() - .collect(), - ) - } - - /// Get the resource that is hosted on chain at the given object ID. - async fn get_remote_resource_info(&self, object_id: ObjectID) -> Result { - let object = get_struct_from_object_response( - &self - .sui_client() - .await? - .read_api() - .get_object_with_options(object_id, SuiObjectDataOptions::new().with_content()) - .await?, - )?; - get_dynamic_field!(object, "value", SuiMoveValue::Struct)?.try_into() - } - async fn sui_client(&self) -> Result { self.wallet.get_client().await } @@ -252,4 +217,11 @@ impl SiteManager { .1 .object_ref()) } + + /// Returns whether the site needs to be transferred to the active address. + /// + /// A new site needs to be transferred to the active address. + fn needs_transfer(&self) -> bool { + matches!(self.site_id, SiteIdentifier::NewSite(_)) + } } diff --git a/site-builder/src/site/resource.rs b/site-builder/src/site/resource.rs index 9efd1a4c..8dfcf5e6 100644 --- a/site-builder/src/site/resource.rs +++ b/site-builder/src/site/resource.rs @@ -18,6 +18,7 @@ use move_core_types::u256::U256; use serde::{Deserialize, Serialize}; use sui_sdk::rpc_types::{SuiMoveStruct, SuiMoveValue}; +use super::SiteData; use crate::{ site::{config::WSResources, content::ContentType}, walrus::{types::BlobId, Walrus}, @@ -74,6 +75,17 @@ pub struct HttpHeaders(pub BTreeMap); impl TryFrom for HttpHeaders { type Error = anyhow::Error; + fn try_from(source: SuiMoveStruct) -> Result { + let map = MapWrapper::try_from(source)?; + Ok(Self(map.0)) + } +} + +pub struct MapWrapper(pub BTreeMap); + +impl TryFrom for MapWrapper { + type Error = anyhow::Error; + fn try_from(source: SuiMoveStruct) -> Result { let contents: Vec<_> = get_dynamic_field!(source, "contents", SuiMoveValue::Vector)?; let mut headers = BTreeMap::new(); @@ -198,19 +210,19 @@ impl<'a> ResourceOp<'a> { /// A summary of the operations performed by the site builder. #[derive(Debug, Clone)] -pub(crate) struct OperationSummary { +pub(crate) struct ResourceOpSummary { operation: String, path: String, blob_id: BlobId, } -impl<'a> From<&ResourceOp<'a>> for OperationSummary { +impl<'a> From<&ResourceOp<'a>> for ResourceOpSummary { fn from(source: &ResourceOp<'a>) -> Self { let (op, info) = match source { ResourceOp::Deleted(resource) => ("deleted".to_owned(), &resource.info), ResourceOp::Created(resource) => ("created".to_owned(), &resource.info), }; - OperationSummary { + ResourceOpSummary { operation: op, path: info.path.clone(), blob_id: info.blob_id, @@ -218,7 +230,7 @@ impl<'a> From<&ResourceOp<'a>> for OperationSummary { } } -impl Display for OperationSummary { +impl Display for ResourceOpSummary { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!( f, @@ -228,11 +240,11 @@ impl Display for OperationSummary { } } -pub(crate) struct OperationsSummary(pub Vec); +pub(crate) struct OperationsSummary(pub Vec); impl<'a> From<&Vec>> for OperationsSummary { fn from(source: &Vec>) -> Self { - Self(source.iter().map(OperationSummary::from).collect()) + Self(source.iter().map(ResourceOpSummary::from).collect()) } } @@ -258,27 +270,28 @@ impl Display for OperationsSummary { } /// A set of resources composing a site. -#[derive(Default, Debug, Clone)] +#[derive(Default, Debug, Clone, PartialEq, Eq)] pub(crate) struct ResourceSet { pub inner: BTreeSet, } impl ResourceSet { + /// Creates an empty resource set. + pub fn empty() -> Self { + Self { + inner: BTreeSet::new(), + } + } + /// Returns a vector of deletion and creation operations to move - /// from the current set to the target set. + /// from the start set to the current set. /// /// The deletions are always before the creation operations, such /// that if two resources have the same path but different /// contents they are first deleted and then created anew. - pub fn diff<'a>(&'a self, target: &'a ResourceSet) -> Vec> { - let create = self - .inner - .difference(&target.inner) - .map(ResourceOp::Created); - let delete = target - .inner - .difference(&self.inner) - .map(ResourceOp::Deleted); + pub fn diff<'a>(&'a self, start: &'a ResourceSet) -> Vec> { + let create = self.inner.difference(&start.inner).map(ResourceOp::Created); + let delete = start.inner.difference(&self.inner).map(ResourceOp::Deleted); delete.chain(create).collect() } @@ -336,11 +349,9 @@ impl Display for ResourceSet { pub(crate) struct ResourceManager { /// The controller for the Walrus CLI. pub walrus: Walrus, - /// The resources in the site. - pub resources: ResourceSet, /// The ws-resources.json contents. pub ws_resources: Option, - /// THe ws-resource file path. + /// The ws-resource file path. pub ws_resources_path: Option, } @@ -352,7 +363,6 @@ impl ResourceManager { ) -> Result { Ok(ResourceManager { walrus, - resources: ResourceSet::default(), ws_resources, ws_resources_path, }) @@ -441,9 +451,13 @@ impl ResourceManager { } /// Recursively iterate a directory and load all [`Resources`][Resource] within. - pub fn read_dir(&mut self, root: &Path) -> Result<()> { - self.resources = ResourceSet::from_iter(self.iter_dir(root, root)?); - Ok(()) + pub fn read_dir(&mut self, root: &Path) -> Result { + Ok(SiteData::new( + ResourceSet::from_iter(self.iter_dir(root, root)?), + self.ws_resources + .as_ref() + .and_then(|config| config.routes.clone()), + )) } fn iter_dir(&self, start: &Path, root: &Path) -> Result> { diff --git a/site-builder/src/summary.rs b/site-builder/src/summary.rs new file mode 100644 index 00000000..69543d03 --- /dev/null +++ b/site-builder/src/summary.rs @@ -0,0 +1,102 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +//! Summaries of the run results. + +use crate::{ + site::{resource::ResourceOp, RouteOps, SiteDataDiff}, + walrus::types::BlobId, +}; + +/// The struct can be turned into a summary. +pub trait Summarizable { + fn to_summary(&self) -> String; +} + +pub struct ResourceOpSummary { + operation: String, + path: String, + blob_id: BlobId, +} + +impl From<&ResourceOp<'_>> for ResourceOpSummary { + fn from(value: &ResourceOp<'_>) -> Self { + let (operation, info) = match value { + ResourceOp::Deleted(resource) => ("deleted".to_owned(), &resource.info), + ResourceOp::Created(resource) => ("created".to_owned(), &resource.info), + }; + ResourceOpSummary { + operation, + path: info.path.clone(), + blob_id: info.blob_id, + } + } +} + +impl Summarizable for ResourceOpSummary { + fn to_summary(&self) -> String { + format!( + "{} resource {} with blob ID {}", + self.operation, self.path, self.blob_id + ) + } +} + +impl Summarizable for RouteOps { + fn to_summary(&self) -> String { + match self { + RouteOps::Unchanged => "The site routes were left unchanged".to_owned(), + RouteOps::Replace(_) => "The site routes were modified".to_owned(), + } + } +} + +impl Summarizable for Vec { + fn to_summary(&self) -> String { + self.iter() + .map(|op| format!(" - {}", op.to_summary())) + .collect::>() + .join("\n") + .to_owned() + } +} + +pub struct SiteDataDiffSummary { + resource_ops: Vec, + route_ops: RouteOps, +} + +impl From<&SiteDataDiff<'_>> for SiteDataDiffSummary { + fn from(value: &SiteDataDiff<'_>) -> Self { + SiteDataDiffSummary { + resource_ops: value.resource_ops.iter().map(|op| op.into()).collect(), + route_ops: value.route_ops.clone(), + } + } +} + +impl From> for SiteDataDiffSummary { + fn from(value: SiteDataDiff<'_>) -> Self { + (&value).into() + } +} + +impl Summarizable for SiteDataDiffSummary { + fn to_summary(&self) -> String { + if self.resource_ops.is_empty() && self.route_ops.is_unchanged() { + return "No operation needs to be performed".to_owned(); + } + + let resource_str = if !self.resource_ops.is_empty() { + format!( + "Resource operations performed:\n{}\n", + self.resource_ops.to_summary() + ) + } else { + "".to_owned() + }; + let route_str = self.route_ops.to_summary(); + + format!("{}{}", resource_str, route_str) + } +} diff --git a/site-builder/src/util.rs b/site-builder/src/util.rs index afa7719a..dda7d074 100644 --- a/site-builder/src/util.rs +++ b/site-builder/src/util.rs @@ -1,7 +1,7 @@ // Copyright (c) Mysten Labs, Inc. // SPDX-License-Identifier: Apache-2.0 -use std::{collections::HashMap, path::PathBuf, str}; +use std::{path::PathBuf, str}; use anyhow::{anyhow, Result}; use futures::Future; @@ -15,26 +15,8 @@ use sui_sdk::{ SuiTransactionBlockEffectsAPI, }, wallet_context::WalletContext, - SuiClient, }; -use sui_types::{ - base_types::{ObjectID, SuiAddress}, - dynamic_field::DynamicFieldInfo, -}; - -pub async fn get_all_dynamic_field_info( - client: &SuiClient, - object_id: ObjectID, -) -> Result> { - let iter = handle_pagination(|cursor| { - client - .read_api() - .get_dynamic_fields(object_id, cursor, None) - }) - .await? - .collect(); - Ok(iter) -} +use sui_types::base_types::{ObjectID, SuiAddress}; pub async fn handle_pagination( closure: F, @@ -134,27 +116,6 @@ pub(crate) fn get_struct_from_object_response( } } -pub async fn get_existing_resource_ids( - client: &SuiClient, - site_id: ObjectID, -) -> Result> { - let info = get_all_dynamic_field_info(client, site_id).await?; - Ok(info - .iter() - .filter_map(|d| get_path_from_info(d).map(|path| (path, d.object_id))) - .collect::>()) -} - -// TODO(giac): check the type of the name. -fn get_path_from_info(info: &DynamicFieldInfo) -> Option { - info.name - .value - .as_object() - .and_then(|obj| obj.get("path")) - .and_then(|p| p.as_str()) - .map(|s| s.to_owned()) -} - /// Returns the path if it is `Some` or any of the default paths if they exist (attempt in order). pub fn path_or_defaults_if_exist(path: &Option, defaults: &[PathBuf]) -> Option { let mut path = path.clone();