feat(portal): add caching interface to sw portal routing

examples/snake/ws-resources.json

@@ -4,5 +4,8 @@
       "Content-Type": "text/html; charset=utf-8",
       "Cache-Control": "max-age=3500"
     }
+  },
+  "routes": {
+    "/path/*": "/file.svg"
   }
 }

move/walrus_site/Move.lock

@@ -2,7 +2,7 @@
 
 [move]
 version = 0
-manifest_digest = "E9C7401E4D5BF8E549B5CB4E935992B4BEEEB50757B900BB03D72491F31A7087"
+manifest_digest = "6955A2973AF0DB196E469AD72FDB6D203278DF6207C4561B7B9FDC4BD1AC951D"
 deps_digest = "F8BBB0CCB2491CA29A3DF03D6F92277A4F3574266507ACD77214D37ECA3F3082"
 
 dependencies = [
@@ -22,7 +22,7 @@ dependencies = [
 ]
 
 [move.toolchain-version]
-compiler-version = "1.33.1"
+compiler-version = "1.34.2"
 edition = "2024.beta"
 flavor = "sui"
 
@@ -36,6 +36,6 @@ published-version = "1"
 
 [env.testnet]
 chain-id = "4c78adac"
-original-published-id = "0xe15cd956d3f54ad0b6608b01b96e9999d66552dfd025e698ac16cd0df1787a25"
-latest-published-id = "0xe15cd956d3f54ad0b6608b01b96e9999d66552dfd025e698ac16cd0df1787a25"
+original-published-id = "0x8a928f3cb7b4b42a957d554acd3928c4fbf5b0052f8218513a259f03837390fc"
+latest-published-id = "0x8a928f3cb7b4b42a957d554acd3928c4fbf5b0052f8218513a259f03837390fc"
 published-version = "1"

move/walrus_site/Move.toml

@@ -1,6 +1,9 @@
 [package]
 name = "walrus_site"
+license = "Apache-2.0"
+authors = ["Mysten Labs <[email protected]>"]
 version = "0.0.1"
+edition = "2024.beta"
 
 [dependencies]
 Sui = { git = "https://github.com/MystenLabs/sui.git", subdir = "crates/sui-framework/packages/sui-framework", rev = "framework/testnet" }

move/walrus_site/sources/site.move

@@ -1,20 +1,23 @@
 /// The module exposes the functionality to create and update Walrus sites.
 module walrus_site::site {
-    use std::option::Option;
-    use sui::object::{Self, UID};
-    use sui::tx_context::TxContext;
     use sui::dynamic_field as df;
     use std::string::String;
     use sui::vec_map;
 
+    /// The name of the dynamic field containing the routes.
+    const ROUTES_FIELD: vector<u8> = b"routes";
+
+    /// An insertion of route was attempted, but the related resource does not exist.
+    const EResourceDoesNotExist: u64 = 0;
+
     /// The site published on Sui.
-    struct Site has key, store {
+    public struct Site has key, store {
         id: UID,
         name: String,
     }
 
     /// A resource in a site.
-    struct Resource has store, drop {
+    public struct Resource has store, drop {
         path: String,
         // Response, Representation and Payload headers
         // regarding the contents of the resource.
@@ -29,10 +32,15 @@ module walrus_site::site {
     /// Representation of the resource path.
     ///
     /// Ensures there are no namespace collisions in the dynamic fields.
-    struct ResourcePath has copy, store, drop {
+    public struct ResourcePath has copy, store, drop {
         path: String,
     }
 
+    /// The routes for a site.
+    public struct Routes has store, drop {
+        route_list: vec_map::VecMap<String, String>,
+    }
+
     /// Creates a new site.
     public fun new_site(name: String, ctx: &mut TxContext): Site {
         Site {
@@ -57,14 +65,10 @@ module walrus_site::site {
 
     /// Adds a header to the Resource's headers vector.
     public fun add_header(resource: &mut Resource, name: String, value: String) {
-        // Will throw an exception if duplicate key.
-        vec_map::insert(
-            &mut resource.headers,
-            name,
-            value
-        );
+        resource.headers.insert(name, value);
     }
 
+    /// Creates a new resource path.
     fun new_path(path: String): ResourcePath {
         ResourcePath { path }
     }
@@ -96,8 +100,58 @@ module walrus_site::site {
 
     /// Changes the path of a resource on a site.
     public fun move_resource(site: &mut Site, old_path: String, new_path: String) {
-        let resource = remove_resource(site, old_path);
+        let mut resource = remove_resource(site, old_path);
         resource.path = new_path;
         add_resource(site, resource);
     }
+
+    // Routes.
+
+    /// Creates a new `Routes` object.
+    fun new_routes(): Routes {
+        Routes { route_list: vec_map::empty() }
+    }
+
+    /// Inserts a route into the `Routes` object.
+    ///
+    /// The insertion operation fails if the route already exists.
+    fun routes_insert(routes: &mut Routes, route: String, resource_path: String) {
+        routes.route_list.insert(route, resource_path);
+    }
+
+    /// Removes a route from the `Routes` object.
+    fun routes_remove(routes: &mut Routes, route: &String): (String, String) {
+        routes.route_list.remove(route)
+    }
+
+    // Routes management on the site.
+
+    /// Add the routes dynamic field to the site.
+    public fun create_routes(site: &mut Site) {
+        let routes = new_routes();
+        df::add(&mut site.id, ROUTES_FIELD, routes);
+    }
+
+    /// Remove all routes from the site.
+    public fun remove_all_routes_if_exist(site: &mut Site): Option<Routes> {
+        df::remove_if_exists(&mut site.id, ROUTES_FIELD)
+    }
+
+    /// Add a route to the site.
+    ///
+    /// The insertion operation fails:
+    /// - if the route already exists; or
+    /// - if the related resource path does not already exist as a dynamic field on the site.
+    public fun insert_route(site: &mut Site, route: String, resource_path: String) {
+        let path_obj = new_path(resource_path);
+        assert!(df::exists_(&site.id, path_obj), EResourceDoesNotExist);
+        let routes = df::borrow_mut(&mut site.id, ROUTES_FIELD);
+        routes_insert(routes, route, resource_path);
+    }
+
+    /// Remove a route from the site.
+    public fun remove_route(site: &mut Site, route: &String): (String, String) {
+        let routes = df::borrow_mut(&mut site.id, ROUTES_FIELD);
+        routes_remove(routes, route)
+    }
 }

portal/common/lib/bcs_data_parsing.ts

@@ -41,3 +41,7 @@ export function DynamicFieldStruct<K, V>(K: BcsType<K>, V: BcsType<V>) {
         value: V,
     });
 }
+
+export const RoutesStruct = bcs.struct("Routes", {
+    routes_list: bcs.map(bcs.string(), bcs.string())
+})

portal/common/lib/constants.ts

@@ -1,16 +1,16 @@
 // Copyright (c) Mysten Labs, Inc.
 // SPDX-License-Identifier: Apache-2.0
 
-export const NETWORK = "testnet"
-export const AGGREGATOR = "https://aggregator-devnet.walrus.space:443"
-export const SITE_PACKAGE = "0xe15cd956d3f54ad0b6608b01b96e9999d66552dfd025e698ac16cd0df1787a25"
-export const MAX_REDIRECT_DEPTH = 3
+export const NETWORK = "testnet";
+export const AGGREGATOR = "https://aggregator-devnet.walrus.space:443";
+export const SITE_PACKAGE = "0xee40a3dddc38e3c9f594f74f85ae3bda75b0ed05c6f2359554126409cf7e8e9d";
+export const MAX_REDIRECT_DEPTH = 3;
 export const SITE_NAMES: { [key: string]: string } = {
     // Any hardcoded (non suins) name -> object_id mappings go here
     // e.g.,
     // landing: "0x1234..."
 };
 // The default portal to redirect to if the browser does not support service workers.
-export const FALLBACK_PORTAL = "blob.store"
+export const FALLBACK_PORTAL = "blob.store";
 // The string representing the ResourcePath struct in the walrus_site package.
 export const RESOURCE_PATH_MOVE_TYPE = SITE_PACKAGE + "::site::ResourcePath";

portal/common/lib/page_fetching.ts

@@ -3,7 +3,7 @@
 
 import { getFullnodeUrl, SuiClient } from "@mysten/sui/client";
 import { NETWORK } from "./constants";
-import { DomainDetails, isResource } from "./types/index";
+import { DomainDetails, isResource, isRoutes } from "./types/index";
 import { subdomainToObjectId, HEXtoBase36 } from "./objectId_operations";
 import { resolveSuiNsAddress, hardcodedSubdmains } from "./suins";
 import { fetchResource } from "./resource";
@@ -16,19 +16,55 @@ import {
 import { aggregatorEndpoint } from "./aggregator";
 import { toB64 } from "@mysten/bcs";
 import { sha256 } from "./crypto";
+import { getRoutes, matchPathToRoute } from "./routing";
+import { RoutingCacheInterface } from "./routing_cache_interface";
+import { Routes, Empty, isEmpty } from "./types/index";
 
 /**
  * Resolves the subdomain to an object ID, and gets the corresponding resources.
  */
-export async function resolveAndFetchPage(parsedUrl: DomainDetails): Promise<Response> {
+export async function resolveAndFetchPage(
+    parsedUrl: DomainDetails, cache?: RoutingCacheInterface
+): Promise<Response> {
     const rpcUrl = getFullnodeUrl(NETWORK);
     const client = new SuiClient({ url: rpcUrl });
     const resolveObjectResult = await resolveObjectId(parsedUrl, client);
     const isObjectId = typeof resolveObjectResult == "string";
     if (isObjectId) {
         console.log("Object ID: ", resolveObjectResult);
         console.log("Base36 version of the object ID: ", HEXtoBase36(resolveObjectResult));
-        return fetchPage(client, resolveObjectResult, parsedUrl.path);
+        // Rerouting based on the contents of the routes object,
+        // constructed using the ws-resource.json.
+        let routes: Routes | Empty | undefined;
+        if (cache) {
+            console.log("Cache provided, fetching the routes object from the cache.")
+            routes = await cache.get(resolveObjectResult);
+            if (!routes) {
+                console.warn("The routes object was not found in the cache.");
+                // The routes object was not found in the cache, so we need to fetch it.
+                routes = await getRoutes(client, resolveObjectResult);
+                await cache.set(resolveObjectResult, routes ? routes : {});
+            }
+            if (isEmpty(routes)) {
+                console.warn("The routes object was already fetched, but it was empty.");
+                return fetchPage(client, resolveObjectResult, parsedUrl.path);
+            }
+        } else {
+            console.warn("No cache provided, fetching the routes object on every request.");
+            routes = await getRoutes(client, resolveObjectResult);
+            if (!routes) {
+                console.warn("No routes found for the object ID");
+                return fetchPage(client, resolveObjectResult, parsedUrl.path);
+            }
+        }
+        let matchingRoute: string | undefined;
+        if (isRoutes(routes)) {
+            matchingRoute = matchPathToRoute(parsedUrl.path, routes)
+            if (!matchingRoute) {
+                console.warn(`No matching route found for ${parsedUrl.path}`);
+            }
+        }
+        return fetchPage(client, resolveObjectResult, matchingRoute ?? parsedUrl.path);
     }
     return resolveObjectResult;
 }
@@ -70,6 +106,8 @@ export async function fetchPage(
     objectId: string,
     path: string,
 ): Promise<Response> {
+    // TODO: On the portal side, fetch the route object, store it to a routes object.
+    // Then pass the routes object to the fetchPage params.
     const result = await fetchResource(client, objectId, path, new Set<string>());
     if (!isResource(result) || !result.blob_id) {
         if (path !== "/404.html") {

portal/common/lib/routing.test.ts

@@ -0,0 +1,42 @@
+// Copyright (c) Mysten Labs, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+import { getRoutes, matchPathToRoute } from "./routing";
+import { test, expect } from 'vitest';
+import { SuiClient, getFullnodeUrl } from "@mysten/sui/client";
+import { NETWORK } from "./constants";
+
+const snakeSiteObjectId = '0x3e01b1b8bf0e54f7843596345faff146f1047e304410ed2eb85d5f67ad404206';
+test.skip('getRoutes', async () => {
+    // TODO: when you make sure get_routes fetches
+    // the Routes dynamic field, mock the request.
+    const rpcUrl = getFullnodeUrl(NETWORK);
+    const client = new SuiClient({ url: rpcUrl });
+    const routes = await getRoutes(client, snakeSiteObjectId);
+    console.log(routes)
+});
+
+const routesExample = {
+    routes_list: new Map<string, string>([
+        ['/*', '/default.html'],
+        ['/somewhere/else', 'else.jpeg'],
+        ['/somewhere/else/*', 'star-else.gif'],
+        ['/path/to/*', '/somewhere.html'],
+    ])
+};
+
+const testCases = [
+    ["/path/to/somewhere/", "/somewhere.html"],
+    ["/somewhere/else", 'else.jpeg'],
+    ["/", "/default.html"],
+    ["/somewhere", "/default.html"],
+    ["/somewhere/else/star", "star-else.gif"],
+    ["/somewhere/else/", 'star-else.gif'],
+]
+
+testCases.forEach(([requestPath, expected]) => {
+    test(`matchPathToRoute: "${requestPath}" -> "${expected}"`, () => {
+        const match = matchPathToRoute(requestPath, routesExample)
+        expect(match).toEqual(expected)
+    })
+});