feat(site-builder): add pre-stored blobs support

site-builder/Cargo.toml

@@ -21,7 +21,7 @@ notify = "6.1.1"
 regex = "1.11.0"
 serde = { version = "1.0.203", features = ["derive"] }
 serde_json = "1.0.117"
-serde_with = { version = "3.8.1", features = ["base64"] }
+serde_with = { version = "3.8.1", features = ["base64", "hex"] }
 serde_yaml = "0.9"
 shared-crypto = { git = "https://github.com/MystenLabs/sui", tag = "testnet-v1.34.2" }
 sui-keys = { git = "https://github.com/MystenLabs/sui", tag = "testnet-v1.34.2" }

site-builder/src/main.rs

@@ -156,11 +156,18 @@ enum Commands {
         object_id: ObjectID,
     },
     /// Show the pages composing the site at the given object ID.
-    Sitemap { object: ObjectID },
+    Sitemap {
+        object: ObjectID,
+    },
     /// Preprocess the directory, creating and linking index files.
     /// This command allows to publish directories as sites. Warning: Rewrites all `index.html`
     /// files.
-    ListDirectory { path: PathBuf },
+    ListDirectory {
+        path: PathBuf,
+    },
+    ListResources {
+        object: ObjectID,
+    },
 }
 
 /// The configuration for the site builder.
@@ -283,9 +290,10 @@ async fn run() -> Result<()> {
         // below will be monitored for changes.
         Commands::Sitemap { object } => {
             let wallet = load_wallet_context(&config.general.wallet)?;
-            let all_dynamic_fields = RemoteSiteFactory::new(&wallet.get_client().await?, object)
-                .get_existing_resources()
-                .await?;
+            let all_dynamic_fields =
+                RemoteSiteFactory::new(&wallet.get_client().await?, config.package)
+                    .get_existing_resources()
+                    .await?;
             println!("Pages in site at object id: {}", object);
             for (name, id) in all_dynamic_fields {
                 println!("  - {:<40} {:?}", name, id);
@@ -295,6 +303,17 @@ async fn run() -> Result<()> {
         Commands::ListDirectory { path } => {
             Preprocessor::preprocess(path.as_path())?;
         }
+        Commands::ListResources { object } => {
+            let wallet = load_wallet_context(&config.general.wallet)?;
+            let pre_built = RemoteSiteFactory::new(&wallet.get_client().await?, config.package)
+                .get_from_chain(object)
+                .await?
+                .to_pre_built();
+            println!(
+                "{{\"pre_built\": {}}}",
+                serde_json::to_string_pretty(&pre_built)?
+            );
+        }
     };
 
     Ok(())

site-builder/src/site.rs

@@ -11,8 +11,10 @@ pub mod resource;
 use std::{collections::HashMap, str::FromStr};
 
 use anyhow::Result;
+use config::LocalResource;
 use contracts::get_sui_object;
 use resource::{ResourceOp, ResourceSet};
+use serde::{Deserialize, Serialize};
 use sui_sdk::SuiClient;
 use sui_types::{base_types::ObjectID, dynamic_field::DynamicFieldInfo, TypeTag};
 
@@ -108,8 +110,25 @@ impl SiteData {
             _ => RouteOps::Unchanged,
         }
     }
+
+    // HACK(giac): use this to get something quick that serializes to something to load to the
+    // pre_built field.
+    pub fn to_pre_built(&self) -> PreBuilt {
+        PreBuilt(
+            self.resources
+                .inner
+                .iter()
+                .cloned()
+                .map(|resource| LocalResource::from(resource.info))
+                .collect(),
+        )
+    }
 }
 
+// HACK(giac): temp, see above.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct PreBuilt(pub Vec<LocalResource>);
+
 /// Fetches remote sites.
 pub struct RemoteSiteFactory<'a> {
     sui_client: &'a SuiClient,

site-builder/src/site/config.rs

@@ -4,10 +4,60 @@
 use std::{collections::BTreeMap, path::Path};
 
 use anyhow::{Context, Result};
-use serde::{Deserialize, Serialize};
+use move_core_types::u256::U256;
+use serde::{Deserialize, Deserializer, Serialize};
+use serde_with::{serde_as, DisplayFromStr};
 
 use super::Routes;
-use crate::types::HttpHeaders;
+use crate::{
+    types::{HttpHeaders, Range, SuiResource},
+    walrus::types::BlobId,
+};
+
+// HACK(giac): this is just to allow easy parsing of local resources.
+/// Information about a resource.
+///
+/// This struct mirrors the information that is stored on chain.
+#[serde_as]
+#[derive(PartialEq, Eq, Debug, Clone, PartialOrd, Ord, Serialize, Deserialize)]
+pub(crate) struct LocalResource {
+    /// The relative path the resource will have on Sui.
+    pub path: String,
+    /// Response, Representation and Payload headers.
+    pub headers: HttpHeaders,
+    /// The blob ID of the resource.
+    #[serde_as(as = "DisplayFromStr")]
+    pub blob_id: BlobId,
+    /// The hash of the blob contents. Serialze and deserialize as hex string.
+    #[serde_as(as = "serde_with::hex::Hex")]
+    pub blob_hash: [u8; 32],
+    /// Byte ranges for the resource.
+    pub range: Option<Range>,
+}
+
+impl From<LocalResource> for SuiResource {
+    fn from(resource: LocalResource) -> Self {
+        SuiResource {
+            path: resource.path,
+            headers: resource.headers,
+            blob_id: resource.blob_id,
+            blob_hash: U256::from_le_bytes(&resource.blob_hash),
+            range: resource.range,
+        }
+    }
+}
+
+impl From<SuiResource> for LocalResource {
+    fn from(resource: SuiResource) -> Self {
+        LocalResource {
+            path: resource.path,
+            headers: resource.headers,
+            blob_id: resource.blob_id,
+            blob_hash: resource.blob_hash.to_le_bytes(),
+            range: resource.range,
+        }
+    }
+}
 
 /// Deserialized object of the file's `ws-resource.json` contents.
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -18,6 +68,25 @@ pub struct WSResources {
     /// The HTTP headers to be set for the resources.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub routes: Option<Routes>,
+    #[serde(
+        default,
+        skip_serializing_if = "Option::is_none",
+        deserialize_with = "deserialize_sui_from_local"
+    )]
+    pub pre_built: Option<Vec<SuiResource>>,
+}
+
+fn deserialize_sui_from_local<'de, D>(deserializer: D) -> Result<Option<Vec<SuiResource>>, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    let resources: Option<Vec<LocalResource>> =
+        Deserialize::deserialize(deserializer).unwrap_or(None);
+    if let Some(res) = resources {
+        Ok(Some(res.into_iter().map(SuiResource::from).collect()))
+    } else {
+        Ok(None)
+    }
 }
 
 impl WSResources {
@@ -37,6 +106,48 @@ impl WSResources {
 mod tests {
 
     use super::*;
+    use crate::walrus::types::BlobId;
+
+    const PRE_BUILT_DATA: &str = r#"
+        "pre_built": [
+            {
+                "path": "/index.html",
+                "headers": {
+                    "Cache-Control": "no-cache",
+                    "Content-Encoding": "gzip",
+                    "Content-Type": "application/json"
+                },
+                "blob_id": "AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQE",
+                "blob_hash": "1212121212121212121212121212121212121212121212121212121212121212",
+                "range": null
+            }
+        ]
+        "#;
+
+    #[test]
+    fn test_deserialize_resource() {
+        let resource = LocalResource {
+            path: "/index.html".to_owned(),
+            blob_hash: [12u8; 32],
+            blob_id: BlobId([1u8; 32]),
+            range: None,
+            headers: HttpHeaders(
+                vec![
+                    ("Content-Type".to_owned(), "application/json".to_owned()),
+                    ("Content-Encoding".to_owned(), "gzip".to_owned()),
+                    ("Cache-Control".to_owned(), "no-cache".to_owned()),
+                ]
+                .into_iter()
+                .collect(),
+            ),
+        };
+
+        let serialized = serde_json::to_string(&resource).expect("serialization should succeed");
+        println!("{}", serialized);
+
+        let _: LocalResource =
+            serde_json::from_str(&serialized).expect("deserialization should succeed");
+    }
 
     const HEADER_DATA: &str = r#"
             "headers": {
@@ -60,7 +171,7 @@ mod tests {
         serde_json::from_str::<WSResources>(&header_data).expect("parsing should succeed");
         let route_data = format!("{{{}}}", ROUTE_DATA);
         serde_json::from_str::<WSResources>(&route_data).expect("parsing should succeed");
-        let route_header_data = format!("{{{},{}}}", HEADER_DATA, ROUTE_DATA);
-        serde_json::from_str::<WSResources>(&route_header_data).expect("parsing should succeed");
+        let pre_built_data = format!("{{{}}}", PRE_BUILT_DATA);
+        serde_json::from_str::<WSResources>(&pre_built_data).expect("parsing should succeed");
     }
 }

site-builder/src/site/resource.rs

@@ -457,6 +457,22 @@ impl ResourceManager {
             resources.inner.extend(resource?);
         }
 
+        // HACK(giac): add the local resources from the config.
+        if let Some(additional) = self
+            .ws_resources
+            .as_ref()
+            .and_then(|config| config.pre_built.as_ref())
+            .map(|pre_built| {
+                pre_built
+                    .clone()
+                    .into_iter()
+                    .map(Resource::from)
+                    .collect::<Vec<_>>()
+            })
+        {
+            resources.inner.extend(additional);
+        }
+
         Ok(SiteData::new(
             resources,
             self.ws_resources
@@ -596,6 +612,7 @@ mod tests {
         Some(WSResources {
             headers: Some(headers),
             routes: None,
+            pre_built: None,
         })
     }
 }

site-builder/src/types.rs

@@ -66,8 +66,20 @@ where
     Ok(HttpHeaders(headers.into_iter().collect()))
 }
 
+/// Serialized the http headers as a vector of tuples, instead of a map.
+///
+/// This is required to serialize in a similar representation as the on-chain struct.
+#[allow(dead_code)]
+fn serialize_http_headers<S>(headers: &HttpHeaders, serializer: S) -> Result<S::Ok, S::Error>
+where
+    S: Serializer,
+{
+    let headers: Vec<_> = headers.0.iter().collect();
+    headers.serialize(serializer)
+}
+
 /// Serialize as string to make sure that the json output uses the base64 encoding.
-fn serialize_blob_id<S>(blob_id: &BlobId, serializer: S) -> Result<S::Ok, S::Error>
+pub fn serialize_blob_id<S>(blob_id: &BlobId, serializer: S) -> Result<S::Ok, S::Error>
 where
     S: Serializer,
 {