Skip to content

Commit

Permalink
feat(push): allow pushing composed components to registry
Browse files Browse the repository at this point in the history 
Signed-off-by: Brian H <[email protected]>
  • Loading branch information
@fibonacci1729
fibonacci1729 committed Jan 30, 2025
1 parent 29ba553 commit 1433ec0
Show file tree
Hide file tree
Showing 5 changed files with 111 additions and 33 deletions.
3 changes: 3 additions & 0 deletions Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 3 additions & 0 deletions crates/oci/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ edition = { workspace = true }
anyhow = { workspace = true }
async-compression = { version = "0.4", features = ["gzip", "tokio"] }
async-tar = "0.5"
async-trait = { workspace = true }
base64 = "0.22"
chrono = "0.4"
# Fork with updated auth to support ACR login
Expand All @@ -22,6 +23,8 @@ reqwest = "0.12"
serde = { workspace = true }
serde_json = { workspace = true }
spin-common = { path = "../common" }
spin-componentize = { path = "../componentize" }
spin-compose = { path = "../compose" }
spin-loader = { path = "../loader" }
spin-locked-app = { path = "../locked-app" }
tempfile = { workspace = true }
Expand Down
132 changes: 99 additions & 33 deletions crates/oci/src/client.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ use std::collections::{BTreeMap, HashMap};
use std::path::{Path, PathBuf};

use anyhow::{bail, Context, Result};
use async_trait::async_trait;
use docker_credential::DockerCredential;
use futures_util::future;
use futures_util::stream::{self, StreamExt, TryStreamExt};
Expand All @@ -18,7 +19,7 @@ use spin_common::ui::quoted_path;
use spin_common::url::parse_file_url;
use spin_loader::cache::Cache;
use spin_loader::FilesMountStrategy;
use spin_locked_app::locked::{ContentPath, ContentRef, LockedApp};
use spin_locked_app::locked::{ContentPath, ContentRef, LockedApp, LockedComponentSource};
use tokio::fs;
use walkdir::WalkDir;

Expand Down Expand Up @@ -119,6 +120,7 @@ impl Client {
reference: impl AsRef<str>,
annotations: Option<BTreeMap<String, String>>,
infer_annotations: InferPredefinedAnnotations,
skip_compose: bool,
) -> Result<Option<String>> {
let reference: Reference = reference
.as_ref()
Expand All @@ -137,8 +139,15 @@ impl Client {
)
.await?;

self.push_locked_core(locked, auth, reference, annotations, infer_annotations)
.await
self.push_locked_core(
locked,
auth,
reference,
annotations,
infer_annotations,
skip_compose,
)
.await
}

/// Push a Spin application to an OCI registry and return the digest (or None
Expand All @@ -149,15 +158,23 @@ impl Client {
reference: impl AsRef<str>,
annotations: Option<BTreeMap<String, String>>,
infer_annotations: InferPredefinedAnnotations,
skip_compose: bool,
) -> Result<Option<String>> {
let reference: Reference = reference
.as_ref()
.parse()
.with_context(|| format!("cannot parse reference {}", reference.as_ref()))?;
let auth = Self::auth(&reference).await?;

self.push_locked_core(locked, auth, reference, annotations, infer_annotations)
.await
self.push_locked_core(
locked,
auth,
reference,
annotations,
infer_annotations,
skip_compose,
)
.await
}

/// Push a Spin application to an OCI registry and return the digest (or None
Expand All @@ -169,10 +186,11 @@ impl Client {
reference: Reference,
annotations: Option<BTreeMap<String, String>>,
infer_annotations: InferPredefinedAnnotations,
skip_compose: bool,
) -> Result<Option<String>> {
let mut locked_app = locked.clone();
let mut layers = self
.assemble_layers(&mut locked_app, AssemblyMode::Simple)
.assemble_layers(&mut locked_app, AssemblyMode::Simple, skip_compose)
.await
.context("could not assemble layers for locked application")?;

Expand All @@ -183,7 +201,7 @@ impl Client {
{
locked_app = locked.clone();
layers = self
.assemble_layers(&mut locked_app, AssemblyMode::Archive)
.assemble_layers(&mut locked_app, AssemblyMode::Archive, skip_compose)
.await
.context("could not assemble archive layers for locked application")?;
}
Expand Down Expand Up @@ -246,43 +264,57 @@ impl Client {
&mut self,
locked: &mut LockedApp,
assembly_mode: AssemblyMode,
skip_compose: bool,
) -> Result<Vec<ImageLayer>> {
let mut layers = Vec::new();
let mut components = Vec::new();
for mut c in locked.clone().components {
// Add the wasm module for the component as layers.
let source = c
.clone()
.source
.content
.source
.context("component loaded from disk should contain a file source")?;

let source = parse_file_url(source.as_str())?;
let layer = Self::wasm_layer(&source).await?;

// Update the module source with the content ref of the layer.
c.source.content = self.content_ref_for_layer(&layer);

layers.push(layer);

let mut deps = BTreeMap::default();
for (dep_name, mut dep) in c.dependencies {
let source = dep
if !skip_compose {
let composed = spin_compose::compose(&ComponentSourceLoader, &c)
.await
.with_context(|| {
format!("failed to resolve dependencies for component {:?}", c.id)
})?;

let layer = ImageLayer::new(composed, WASM_LAYER_MEDIA_TYPE.to_string(), None);
c.source.content = self.content_ref_for_layer(&layer);
c.dependencies.clear();
layers.push(layer);
} else {
// Add the wasm module for the component as layers.
let source = c
.clone()
.source
.content
.source
.context("dependency loaded from disk should contain a file source")?;
let source = parse_file_url(source.as_str())?;
.context("component loaded from disk should contain a file source")?;

let source = parse_file_url(source.as_str())?;
let layer = Self::wasm_layer(&source).await?;

dep.source.content = self.content_ref_for_layer(&layer);
deps.insert(dep_name, dep);
// Update the module source with the content ref of the layer.
c.source.content = self.content_ref_for_layer(&layer);

layers.push(layer);

let mut deps = BTreeMap::default();
for (dep_name, mut dep) in c.dependencies {
let source = dep
.source
.content
.source
.context("dependency loaded from disk should contain a file source")?;
let source = parse_file_url(source.as_str())?;

let layer = Self::wasm_layer(&source).await?;

dep.source.content = self.content_ref_for_layer(&layer);
deps.insert(dep_name, dep);

layers.push(layer);
}
c.dependencies = deps;
}
c.dependencies = deps;

let mut files = Vec::new();
for f in c.files {
Expand Down Expand Up @@ -669,6 +701,32 @@ impl Client {
}
}

struct ComponentSourceLoader;

#[async_trait]
impl spin_compose::ComponentSourceLoader for ComponentSourceLoader {
async fn load_component_source(
&self,
source: &LockedComponentSource,
) -> anyhow::Result<Vec<u8>> {
let source = source
.content
.source
.as_ref()
.context("component loaded from disk should contain a file source")?;

let source = parse_file_url(source.as_str())?;

let bytes = fs::read(&source)
.await
.with_context(|| format!("cannot read wasm module {}", quoted_path(source)))?;

let component = spin_componentize::componentize_if_necessary(&bytes)?;

Ok(component.into())
}
}

/// Unpack contents of the provided archive layer, represented by bytes and its
/// corresponding digest, into the provided cache.
/// A temporary staging directory is created via tempfile::tempdir() to store
Expand Down Expand Up @@ -946,6 +1004,7 @@ mod test {
locked_components: Vec<LockedComponent>,
expected_layer_count: usize,
expected_error: Option<&'static str>,
skip_compose: bool,
}

let tests: Vec<TestCase> = [
Expand All @@ -968,6 +1027,7 @@ mod test {
}}]),
expected_layer_count: 2,
expected_error: None,
skip_compose: true,
},
TestCase {
name: "One component layer and two file layers",
Expand All @@ -992,6 +1052,7 @@ mod test {
}]),
expected_layer_count: 3,
expected_error: None,
skip_compose: true,
},
TestCase {
name: "One component layer and one file with inlined content",
Expand All @@ -1012,6 +1073,7 @@ mod test {
}]),
expected_layer_count: 1,
expected_error: None,
skip_compose: true,
},
TestCase {
name: "One component layer and one dependency component layer",
Expand All @@ -1036,6 +1098,7 @@ mod test {
}]),
expected_layer_count: 2,
expected_error: None,
skip_compose: true,
},
TestCase {
name: "Component has no source",
Expand All @@ -1050,6 +1113,7 @@ mod test {
}]),
expected_layer_count: 0,
expected_error: Some("Invalid URL: \"\""),
skip_compose: true,
},
TestCase {
name: "Duplicate component sources",
Expand All @@ -1070,6 +1134,7 @@ mod test {
}}]),
expected_layer_count: 1,
expected_error: None,
skip_compose: true,
},
TestCase {
name: "Duplicate file paths",
Expand Down Expand Up @@ -1107,6 +1172,7 @@ mod test {
}]),
expected_layer_count: 4,
expected_error: None,
skip_compose: true,
},
]
.to_vec();
Expand Down Expand Up @@ -1137,7 +1203,7 @@ mod test {
assert_eq!(
e,
client
.assemble_layers(&mut locked, AssemblyMode::Simple)
.assemble_layers(&mut locked, AssemblyMode::Simple, tc.skip_compose)
.await
.unwrap_err()
.to_string(),
Expand All @@ -1149,7 +1215,7 @@ mod test {
assert_eq!(
tc.expected_layer_count,
client
.assemble_layers(&mut locked, AssemblyMode::Simple)
.assemble_layers(&mut locked, AssemblyMode::Simple, tc.skip_compose)
.await
.unwrap()
.len(),
Expand Down
5 changes: 5 additions & 0 deletions src/commands/registry.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,10 @@ pub struct Push {
)]
pub insecure: bool,

/// Skip composing the application's components before pushing it.
#[clap(long = "skip-compose", env = PUSH_ALWAYS_SKIP_COMPOSE)]
pub skip_compose: bool,

/// Specifies to perform `spin build` before pushing the application.
#[clap(long, takes_value = false, env = ALWAYS_BUILD_ENV)]
pub build: bool,
Expand Down Expand Up @@ -94,6 +98,7 @@ impl Push {
&self.reference,
annotations,
InferPredefinedAnnotations::All,
self.skip_compose,
)
.await?;
match digest {
Expand Down
1 change: 1 addition & 0 deletions src/opts.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,3 +13,4 @@ pub const WATCH_CLEAR_OPT: &str = "CLEAR";
pub const WATCH_DEBOUNCE_OPT: &str = "DEBOUNCE";
pub const WATCH_SKIP_BUILD_OPT: &str = "SKIP_BUILD";
pub const ALWAYS_BUILD_ENV: &str = "SPIN_ALWAYS_BUILD";
pub const PUSH_ALWAYS_SKIP_COMPOSE: &str = "SPIN_PUSH_ALWAYS_SKIP_COMPOSE";

0 comments on commit 1433ec0

Please sign in to comment.