Skip to content

Commit

Permalink
feat: return full b58/blake3 content hash, verification how-to in des…
Browse files Browse the repository at this point in the history 
…cription, config module
  • Loading branch information
@ozwaldorf
ozwaldorf committed Oct 10, 2024
1 parent 2ba4710 commit 23ae4ec
Showing 1 changed file with 53 additions and 49 deletions.
102 changes: 53 additions & 49 deletions src/main.rs
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
use std::io::{BufRead, Write};
use std::time::{Duration, SystemTime};
use std::time::SystemTime;

use fastly::handle::ResponseHandle;
use fastly::http::Method;
use fastly::kv_store::InsertMode;
use fastly::{cache, Error, KVStore, Request, Response};
Expand All @@ -10,18 +9,24 @@ use humantime::format_duration;
use serde_json::json;
use tinytemplate::TinyTemplate;

/// Upload ID length, up to 64 bytes
const ID_LENGTH: usize = 8;
/// Minimum content size in bytes
const MIN_CONTENT_SIZE: usize = 32;
/// Maximum content size in bytes
const MAX_CONTENT_SIZE: usize = 24 << 20;
/// Fastly key-value storage name
const KV_STORE: &str = "upldis storage";
/// TTL for content
const KV_TTL: Duration = Duration::from_secs(7 * 86400);
/// Request cache ttl
const CACHE_TTL: Duration = Duration::from_secs(30 * 86400);
mod config {
use std::time::Duration;

/// Upload ID length, up to 64 bytes
pub const ID_LENGTH: usize = 8;
/// Minimum content size in bytes
pub const MIN_CONTENT_SIZE: usize = 32;
/// Maximum content size in bytes
pub const MAX_CONTENT_SIZE: usize = 24 << 20;
/// Fastly key-value storage name
pub const KV_STORE: &str = "upldis storage";
/// TTL for content
pub const KV_TTL: Duration = Duration::from_secs(7 * 86400);
/// Request cache ttl
pub const CACHE_TTL: Duration = Duration::from_secs(30 * 86400);
/// Key to store upload metrics under
pub const UPLOAD_METRICS_KEY: &str = "_upload_metrics";
}

/// Helptext template (based on request hostname)
const HELP_TEMPLATE: &str = "\
Expand Down Expand Up @@ -52,6 +57,11 @@ const HELP_TEMPLATE: &str = "\
Once deleted, the content will remain available in regions that
have it cached still.
Content can be verified by using the HTTP header `X-CONTENT-HASH`
which contains the full base58-encoded blake3 hash, the same from
the download URL. Simply ensure the hash provided matches the URL
and the hash of the content you received.
NOTES
* Maximum file size : {max_size}
* Storage TTL : {kv_ttl}
Expand Down Expand Up @@ -90,17 +100,6 @@ fn main(req: Request) -> Result<Response, Error> {
// Filter request methods...
match (req.get_method(), req.get_path()) {
(&Method::GET, "/") => handle_usage(req),
(&Method::GET, "/metrics") => {
let kv = KVStore::open(KV_STORE)?.unwrap();
if let Ok(mut metrics) = kv.lookup("_upload_metrics") {
Ok(Response::from_handles(
ResponseHandle::new(),
metrics.take_body().into_handle(),
))
} else {
Ok(Response::new())
}
}
(&Method::GET, _) => handle_get(req),
(&Method::PUT, _) => handle_put(req),
_ => Ok(Response::from_status(403).with_body("invalid request")),
Expand All @@ -121,7 +120,7 @@ fn handle_usage(req: Request) -> Result<Response, Error> {
2
});

let kv = KVStore::open(KV_STORE)?.expect("kv store to exist");
let kv = KVStore::open(config::KV_STORE)?.expect("kv store to exist");
let upload_counter = get_upload_count(&kv);

// Render template
Expand All @@ -133,9 +132,9 @@ fn handle_usage(req: Request) -> Result<Response, Error> {
"host": host,
"host_caps": host.to_uppercase(),
"padding": padding,
"max_size": *humanize_bytes_binary!(MAX_CONTENT_SIZE),
"kv_ttl": format_duration(KV_TTL).to_string(),
"cache_ttl": format_duration(CACHE_TTL).to_string(),
"max_size": *humanize_bytes_binary!(config::MAX_CONTENT_SIZE),
"kv_ttl": format_duration(config::KV_TTL).to_string(),
"cache_ttl": format_duration(config::CACHE_TTL).to_string(),
"upload_counter": upload_counter
}),
)?;
Expand All @@ -150,10 +149,10 @@ fn handle_put(mut req: Request) -> Result<Response, Error> {
return Ok(Response::from_status(400).with_body_text_plain("missing upload body"));
}
let body = req.take_body_bytes();
if body.len() < MIN_CONTENT_SIZE {
if body.len() < config::MIN_CONTENT_SIZE {
return Ok(Response::from_status(400).with_body_text_plain("content too small"));
}
if body.len() > MAX_CONTENT_SIZE {
if body.len() > config::MAX_CONTENT_SIZE {
return Ok(Response::from_status(413).with_body_text_plain("content too large"));
}

Expand All @@ -166,14 +165,18 @@ fn handle_put(mut req: Request) -> Result<Response, Error> {
.and_then(|v| (!v.is_empty()).then_some(v));

// Hash content and use base58 for the id
let hash = bs58::encode(blake3::hash(&body).as_bytes()).into_string();
let id = &hash[..ID_LENGTH];
let hash = blake3::hash(&body);
let base = bs58::encode(hash.as_bytes()).into_string();
let id = &base[..config::ID_LENGTH];
let key = &format!("file_{id}");

// Insert content to key value store
let kv = KVStore::open(KV_STORE)?.expect("kv store to exist");
let kv = KVStore::open(config::KV_STORE)?.expect("kv store to exist");
if kv.lookup(key).is_err() {
kv.build_insert().time_to_live(KV_TTL).execute(key, body)?;
kv.build_insert()
.metadata(&hash.to_hex())
.time_to_live(config::KV_TTL)
.execute(key, body)?;
track_upload(&kv, id, filename.unwrap_or("undefined"))?;
}

Expand All @@ -190,42 +193,43 @@ fn handle_get(req: Request) -> Result<Response, Error> {
// Extract id from url
let mut segments = req.get_path().split('/').skip(1);
let id = segments.next().expect("empty path is handled earlier");
if id.len() != ID_LENGTH {
if id.len() != config::ID_LENGTH {
return Ok(Response::from_status(404).with_body("not found"));
}
let key = &format!("file_{id}");

// Try to find content in cache
if let Some(found) = cache::core::lookup(key.to_owned().into()).execute()? {
let body_handle = found.to_stream()?.into_handle();
let res = Response::from_handles(ResponseHandle::new(), body_handle);
return Ok(res);
return Ok(Response::new()
.with_header("X-CONTENT-HASH", found.user_metadata().as_ref())
.with_body(found.to_stream()?.into_handle()));
}

// Otherwise, get content from key value store (origin)
let kv = KVStore::open(KV_STORE)?.expect("kv store to exist");
let content = match kv.lookup(key) {
let kv = KVStore::open(config::KV_STORE)?.expect("kv store to exist");
let (meta, content) = match kv.lookup(key) {
Err(_) => return Ok(Response::from_status(404).with_body("not found")),
Ok(mut res) => res.take_body_bytes(),
Ok(mut res) => (res.metadata().unwrap(), res.take_body_bytes()),
};

// Build response with content hash header
let res = Response::new().with_header("X-CONTENT-HASH", meta.as_ref());

// Write content to cache
let mut w = cache::core::insert(key.to_owned().into(), CACHE_TTL)
let mut w = cache::core::insert(key.to_owned().into(), config::CACHE_TTL)
.surrogate_keys(["get"])
.user_metadata(meta)
.execute()?;
w.write_all(&content)?;
w.finish()?;

// Respond with content
Ok(Response::from_body(content))
Ok(res.with_body(content))
}

/// Key to store upload metrics under
const UPLOAD_METRICS_KEY: &str = "_upload_metrics";

/// Get upload count from the metadata, or fallback to the number of metric lines.
fn get_upload_count(kv: &KVStore) -> usize {
kv.lookup(UPLOAD_METRICS_KEY)
kv.lookup(config::UPLOAD_METRICS_KEY)
.ok()
.map(|mut v| {
v.metadata()
Expand All @@ -244,7 +248,7 @@ fn track_upload(kv: &KVStore, id: &str, file: &str) -> Result<(), Error> {
.mode(InsertMode::Append)
.metadata(&new_count.to_string())
.execute(
UPLOAD_METRICS_KEY,
config::UPLOAD_METRICS_KEY,
format!(
"{:?} , {id} , {file}\n",
SystemTime::now()
Expand Down

0 comments on commit 23ae4ec

Please sign in to comment.