From 90e2cede7ecee776279083513006d5bc004793eb Mon Sep 17 00:00:00 2001 From: Jake Goulding Date: Thu, 23 Nov 2023 21:50:24 -0500 Subject: [PATCH 1/3] Migrate crate reporting to the orchestrator Goodbye, sandbox. --- compiler/base/orchestrator/Cargo.lock | 24 ++ compiler/base/orchestrator/Cargo.toml | 1 + compiler/base/orchestrator/src/coordinator.rs | 60 +++++ ui/Cargo.lock | 1 + ui/src/main.rs | 24 +- ui/src/metrics.rs | 25 -- ui/src/sandbox.rs | 220 +----------------- ui/src/server_axum.rs | 85 ++----- 8 files changed, 113 insertions(+), 327 deletions(-) diff --git a/compiler/base/orchestrator/Cargo.lock b/compiler/base/orchestrator/Cargo.lock index f01de3890..d3aa01369 100644 --- a/compiler/base/orchestrator/Cargo.lock +++ b/compiler/base/orchestrator/Cargo.lock @@ -228,6 +228,12 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "itoa" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" + [[package]] name = "lazy_static" version = "1.4.0" @@ -318,6 +324,7 @@ dependencies = [ "modify-cargo-toml", "once_cell", "serde", + "serde_json", "snafu", "tempdir", "tokio", @@ -475,6 +482,12 @@ version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +[[package]] +name = "ryu" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" + [[package]] name = "serde" version = "1.0.192" @@ -495,6 +508,17 @@ dependencies = [ "syn 2.0.39", ] +[[package]] +name = "serde_json" +version = "1.0.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" +dependencies = [ + "itoa", + "ryu", + "serde", +] + [[package]] name = "serde_spanned" version = "0.6.4" diff --git a/compiler/base/orchestrator/Cargo.toml b/compiler/base/orchestrator/Cargo.toml index 5f720ce29..f79af0d14 100644 --- a/compiler/base/orchestrator/Cargo.toml +++ b/compiler/base/orchestrator/Cargo.toml @@ -12,6 +12,7 @@ bincode = { version = "1.3", default-features = false } futures = { version = "0.3.28", default-features = false, features = ["executor"] } modify-cargo-toml = { path = "../modify-cargo-toml", default-features = false } serde = { version = "1.0", default-features = false, features = ["derive"] } +serde_json = { version = "1.0.108", default-features = false, features = ["std"] } snafu = { version = "0.7.4", default-features = false, features = ["futures", "std"] } tokio = { version = "1.28", default-features = false, features = ["fs", "io-std", "io-util", "macros", "process", "rt", "time", "sync"] } tokio-stream = { version = "0.1.14", default-features = false } diff --git a/compiler/base/orchestrator/src/coordinator.rs b/compiler/base/orchestrator/src/coordinator.rs index 2e4cf73f1..4f84a4213 100644 --- a/compiler/base/orchestrator/src/coordinator.rs +++ b/compiler/base/orchestrator/src/coordinator.rs @@ -2,6 +2,7 @@ use futures::{ future::{BoxFuture, OptionFuture}, Future, FutureExt, }; +use serde::Deserialize; use snafu::prelude::*; use std::{ collections::HashMap, @@ -163,6 +164,48 @@ pub enum VersionError { TaskPanic { source: tokio::task::JoinError }, } +#[derive(Debug, Clone)] +pub struct Crate { + pub name: String, + pub version: String, + pub id: String, +} + +#[derive(Deserialize)] +struct InternalCrate { + name: String, + version: String, + id: String, +} + +impl From for Crate { + fn from(other: InternalCrate) -> Self { + let InternalCrate { name, version, id } = other; + Self { name, version, id } + } +} + +#[derive(Debug, Snafu)] +pub enum CratesError { + #[snafu(display("Could not start the container"))] + #[snafu(context(false))] + Start { source: Error }, + + #[snafu(context(false))] // transparent + Container { source: ContainerCratesError }, +} + +#[derive(Debug, Snafu)] +pub enum ContainerCratesError { + #[snafu(display("Could not read the crate information file"))] + #[snafu(context(false))] + Read { source: CommanderError }, + + #[snafu(display("Could not parse the crate information file"))] + #[snafu(context(false))] + Deserialization { source: serde_json::Error }, +} + #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub enum AssemblyFlavor { Att, @@ -789,6 +832,14 @@ where }) } + pub async fn crates(&self) -> Result, CratesError> { + self.select_channel(Channel::Stable) + .await? + .crates() + .await + .map_err(Into::into) + } + pub async fn execute( &self, request: ExecuteRequest, @@ -1120,6 +1171,15 @@ impl Container { Ok(if o.success { Some(o.stdout) } else { None }) } + async fn crates(&self) -> Result, ContainerCratesError> { + let read = ReadFileRequest { + path: "crate-information.json".into(), + }; + let read = self.commander.one(read).await?; + let crates = serde_json::from_slice::>(&read.0)?; + Ok(crates.into_iter().map(Into::into).collect()) + } + async fn execute( &self, request: ExecuteRequest, diff --git a/ui/Cargo.lock b/ui/Cargo.lock index aaa6c0e81..3132e6ad5 100644 --- a/ui/Cargo.lock +++ b/ui/Cargo.lock @@ -893,6 +893,7 @@ dependencies = [ "futures", "modify-cargo-toml", "serde", + "serde_json", "snafu", "tokio", "tokio-stream", diff --git a/ui/src/main.rs b/ui/src/main.rs index 1fbcf1d52..870a2d106 100644 --- a/ui/src/main.rs +++ b/ui/src/main.rs @@ -162,10 +162,6 @@ impl MetricsToken { #[derive(Debug, Snafu)] enum Error { - #[snafu(display("Sandbox creation failed: {}", source))] - SandboxCreation { source: sandbox::Error }, - #[snafu(display("Caching operation failed: {}", source))] - Caching { source: sandbox::Error }, #[snafu(display("Gist creation failed: {}", source))] GistCreation { source: octocrab::Error }, #[snafu(display("Gist loading failed: {}", source))] @@ -219,6 +215,11 @@ enum Error { #[snafu(display("The WebSocket worker panicked: {}", text))] WebSocketTaskPanic { text: String }, + #[snafu(display("Unable to find the available crates"))] + Crates { + source: orchestrator::coordinator::CratesError, + }, + #[snafu(display("Unable to find the available versions"))] Versions { source: orchestrator::coordinator::VersionsError, @@ -464,21 +465,6 @@ struct EvaluateResponse { error: Option, } -impl From> for MetaCratesResponse { - fn from(me: Vec) -> Self { - let crates = me - .into_iter() - .map(|cv| CrateInformation { - name: cv.name, - version: cv.version, - id: cv.id, - }) - .collect(); - - MetaCratesResponse { crates } - } -} - impl From for MetaGistResponse { fn from(me: gist::Gist) -> Self { MetaGistResponse { diff --git a/ui/src/metrics.rs b/ui/src/metrics.rs index e1da4f64c..239014144 100644 --- a/ui/src/metrics.rs +++ b/ui/src/metrics.rs @@ -10,8 +10,6 @@ use std::{ time::{Duration, Instant}, }; -use crate::sandbox; - lazy_static! { pub(crate) static ref REQUESTS: HistogramVec = register_histogram_vec!( "playground_request_duration_seconds", @@ -204,29 +202,6 @@ impl Labels { } } -pub(crate) trait GenerateLabels { - fn generate_labels(&self, outcome: Outcome) -> Labels; -} - -impl GenerateLabels for &'_ T -where - T: GenerateLabels, -{ - fn generate_labels(&self, outcome: Outcome) -> Labels { - T::generate_labels(self, outcome) - } -} - -pub(crate) trait SuccessDetails: Sized { - fn success_details(&self) -> Outcome; -} - -impl SuccessDetails for Vec { - fn success_details(&self) -> Outcome { - Outcome::Success - } -} - pub(crate) async fn track_metric_no_request_async( endpoint: Endpoint, body: B, diff --git a/ui/src/sandbox.rs b/ui/src/sandbox.rs index 27539aa85..ea90d3877 100644 --- a/ui/src/sandbox.rs +++ b/ui/src/sandbox.rs @@ -1,221 +1,3 @@ -use serde_derive::Deserialize; -use snafu::prelude::*; -use std::{io, time::Duration}; -use tempfile::TempDir; -use tokio::{process::Command, time}; +use std::time::Duration; pub(crate) const DOCKER_PROCESS_TIMEOUT_SOFT: Duration = Duration::from_secs(10); -const DOCKER_PROCESS_TIMEOUT_HARD: Duration = Duration::from_secs(12); - -#[derive(Debug, Deserialize)] -struct CrateInformationInner { - name: String, - version: String, - id: String, -} - -#[derive(Debug, Clone)] -pub struct CrateInformation { - pub name: String, - pub version: String, - pub id: String, -} - -impl From for CrateInformation { - fn from(me: CrateInformationInner) -> Self { - let CrateInformationInner { name, version, id } = me; - Self { name, version, id } - } -} - -#[derive(Debug, Snafu)] -pub enum Error { - #[snafu(display("Unable to create temporary directory: {}", source))] - UnableToCreateTempDir { source: io::Error }, - - #[snafu(display("Unable to start the compiler: {}", source))] - UnableToStartCompiler { source: io::Error }, - #[snafu(display("Unable to find the compiler ID"))] - MissingCompilerId, - #[snafu(display("Unable to wait for the compiler: {}", source))] - UnableToWaitForCompiler { source: io::Error }, - #[snafu(display("Unable to get output from the compiler: {}", source))] - UnableToGetOutputFromCompiler { source: io::Error }, - #[snafu(display("Unable to remove the compiler: {}", source))] - UnableToRemoveCompiler { source: io::Error }, - #[snafu(display("Compiler execution took longer than {} ms", timeout.as_millis()))] - CompilerExecutionTimedOut { - source: tokio::time::error::Elapsed, - timeout: Duration, - }, - - #[snafu(display("Unable to read crate information: {}", source))] - UnableToParseCrateInformation { source: ::serde_json::Error }, -} - -pub type Result = ::std::result::Result; - -macro_rules! docker_command { - ($($arg:expr),* $(,)?) => ({ - let mut cmd = Command::new("docker"); - $( cmd.arg($arg); )* - cmd - }); -} - -fn basic_secure_docker_command() -> Command { - let mut cmd = docker_command!( - "run", - "--platform", - "linux/amd64", - "--detach", - "--cap-drop=ALL", - // Needed to allow overwriting the file - "--cap-add=DAC_OVERRIDE", - "--security-opt=no-new-privileges", - "--workdir", - "/playground", - "--net", - "none", - "--memory", - "512m", - "--memory-swap", - "640m", - "--env", - format!( - "PLAYGROUND_TIMEOUT={}", - DOCKER_PROCESS_TIMEOUT_SOFT.as_secs() - ), - "--oom-score-adj", - "1000", - ); - - if cfg!(feature = "fork-bomb-prevention") { - cmd.args(&["--pids-limit", "512"]); - } - - cmd.kill_on_drop(true); - - cmd -} - -pub struct Sandbox { - #[allow(dead_code)] - scratch: TempDir, -} - -impl Sandbox { - pub async fn new() -> Result { - // `TempDir` performs *synchronous* filesystem operations - // now and when it's dropped. We accept that under the - // assumption that the specific operations will be quick - // enough. - let scratch = tempfile::Builder::new() - .prefix("playground") - .tempdir() - .context(UnableToCreateTempDirSnafu)?; - - Ok(Sandbox { scratch }) - } - - pub async fn crates(&self) -> Result> { - let mut command = basic_secure_docker_command(); - command.args(&[Channel::Stable.container_name()]); - command.args(&["cat", "crate-information.json"]); - - let output = run_command_with_timeout(command).await?; - - let crate_info: Vec = - ::serde_json::from_slice(&output.stdout).context(UnableToParseCrateInformationSnafu)?; - - let crates = crate_info.into_iter().map(Into::into).collect(); - - Ok(crates) - } -} - -async fn run_command_with_timeout(mut command: Command) -> Result { - use std::os::unix::process::ExitStatusExt; - - let timeout = DOCKER_PROCESS_TIMEOUT_HARD; - - let output = command.output().await.context(UnableToStartCompilerSnafu)?; - - // Exit early, in case we don't have the container - if !output.status.success() { - return Ok(output); - } - - let output = String::from_utf8_lossy(&output.stdout); - let id = output - .lines() - .next() - .context(MissingCompilerIdSnafu)? - .trim(); - - // ---------- - - let mut command = docker_command!("wait", id); - - let timed_out = match time::timeout(timeout, command.output()).await { - Ok(Ok(o)) => { - // Didn't time out, didn't fail to run - let o = String::from_utf8_lossy(&o.stdout); - let code = o - .lines() - .next() - .unwrap_or("") - .trim() - .parse() - .unwrap_or(i32::MAX); - Ok(ExitStatusExt::from_raw(code)) - } - Ok(e) => return e.context(UnableToWaitForCompilerSnafu), // Failed to run - Err(e) => Err(e), // Timed out - }; - - // ---------- - - let mut command = docker_command!("logs", id); - let mut output = command - .output() - .await - .context(UnableToGetOutputFromCompilerSnafu)?; - - // ---------- - - let mut command = docker_command!( - "rm", // Kills container if still running - "--force", id - ); - command.stdout(std::process::Stdio::null()); - command - .status() - .await - .context(UnableToRemoveCompilerSnafu)?; - - let code = timed_out.context(CompilerExecutionTimedOutSnafu { timeout })?; - - output.status = code; - - Ok(output) -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, strum::IntoStaticStr)] -pub enum Channel { - Stable, - Beta, - Nightly, -} - -impl Channel { - fn container_name(&self) -> &'static str { - use self::Channel::*; - - match *self { - Stable => "rust-stable", - Beta => "rust-beta", - Nightly => "rust-nightly", - } - } -} diff --git a/ui/src/server_axum.rs b/ui/src/server_axum.rs index 17acf3ee9..bd86b6a25 100644 --- a/ui/src/server_axum.rs +++ b/ui/src/server_axum.rs @@ -4,14 +4,14 @@ use crate::{ record_metric, track_metric_no_request_async, Endpoint, HasLabelsCore, Outcome, UNAVAILABLE_WS, }, - sandbox::{Sandbox, DOCKER_PROCESS_TIMEOUT_SOFT}, - CachingSnafu, ClippyRequest, ClippyResponse, ClippySnafu, CompileRequest, CompileResponse, - CompileSnafu, Config, Error, ErrorJson, EvaluateRequest, EvaluateResponse, EvaluateSnafu, + sandbox::DOCKER_PROCESS_TIMEOUT_SOFT, + ClippyRequest, ClippyResponse, ClippySnafu, CompileRequest, CompileResponse, CompileSnafu, + Config, CratesSnafu, Error, ErrorJson, EvaluateRequest, EvaluateResponse, EvaluateSnafu, ExecuteRequest, ExecuteResponse, ExecuteSnafu, FormatRequest, FormatResponse, FormatSnafu, GhToken, GistCreationSnafu, GistLoadingSnafu, MacroExpansionRequest, MacroExpansionResponse, MacroExpansionSnafu, MetaCratesResponse, MetaGistCreateRequest, MetaGistResponse, MetaVersionResponse, MetricsToken, MiriRequest, MiriResponse, MiriSnafu, MiriVersionSnafu, - Result, SandboxCreationSnafu, ShutdownCoordinatorSnafu, TimeoutSnafu, VersionsSnafu, + Result, ShutdownCoordinatorSnafu, TimeoutSnafu, VersionsSnafu, }; use async_trait::async_trait; use axum::{ @@ -346,7 +346,6 @@ async fn meta_crates( Extension(cache): Extension>, if_none_match: Option>, ) -> Result { - // Json Result> { + let coordinator = Coordinator::new_docker().await; self.crates - .fetch( - |sandbox| async move { Ok(sandbox.crates().await.context(CachingSnafu)?.into()) }, - ) + .fetch(|| async { Ok(coordinator.crates().await.context(CratesSnafu)?.into()) }) .await } @@ -572,7 +570,7 @@ impl SandboxCache { let coordinator = Coordinator::new_docker().await; self.versions - .fetch2(|| async { + .fetch(|| async { Ok(Arc::new( coordinator.versions().await.context(VersionsSnafu)?, )) @@ -633,7 +631,7 @@ where { async fn fetch(&self, generator: F) -> Result> where - F: FnOnce(Sandbox) -> FFut, + F: FnOnce() -> FFut, FFut: Future>, { let data = &mut *self.0.lock().await; @@ -650,60 +648,6 @@ where } async fn set_value(data: &mut Option>, generator: F) -> Result> - where - F: FnOnce(Sandbox) -> FFut, - FFut: Future>, - { - let sandbox = Sandbox::new().await.context(SandboxCreationSnafu)?; - let value = generator(sandbox).await?; - - let old_info = data.take(); - let new_info = CacheInfo::build(value); - - let info = match old_info { - Some(mut old_value) => { - if old_value.value == new_info.value { - // The value hasn't changed; record that we have - // checked recently, but keep the creation time to - // preserve caching. - old_value.validation_time = new_info.validation_time; - old_value - } else { - new_info - } - } - None => new_info, - }; - - let value = info.stamped_value(); - - *data = Some(info); - - Ok(value) - } - - async fn fetch2(&self, generator: F) -> Result> - where - F: FnOnce() -> FFut, - FFut: Future>, - { - let data = &mut *self.0.lock().await; - match data { - Some(info) => { - if info.validation_time.elapsed() <= SANDBOX_CACHE_TIME_TO_LIVE { - Ok(info.stamped_value()) - } else { - Self::set_value2(data, generator).await - } - } - None => Self::set_value2(data, generator).await, - } - } - - async fn set_value2( - data: &mut Option>, - generator: F, - ) -> Result> where F: FnOnce() -> FFut, FFut: Future>, @@ -812,6 +756,19 @@ pub(crate) mod api_orchestrator_integration_impls { use snafu::prelude::*; use std::convert::TryFrom; + impl From> for crate::MetaCratesResponse { + fn from(other: Vec) -> Self { + let crates = other + .into_iter() + .map(|c| { + let Crate { name, version, id } = c; + crate::CrateInformation { name, version, id } + }) + .collect(); + Self { crates } + } + } + impl From<&Version> for crate::MetaVersionResponse { fn from(other: &Version) -> Self { Self { From e82cdbfbeef9942f209a150df83eb7470b92c935 Mon Sep 17 00:00:00 2001 From: Jake Goulding Date: Tue, 21 Nov 2023 11:49:43 -0500 Subject: [PATCH 2/3] Upgrade test semver dependencies --- tests/Gemfile.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/Gemfile.lock b/tests/Gemfile.lock index 0bee7a9b2..386049bd3 100644 --- a/tests/Gemfile.lock +++ b/tests/Gemfile.lock @@ -21,11 +21,11 @@ GEM matrix (0.4.2) mini_mime (1.1.5) mini_portile2 (2.8.5) - nokogiri (1.15.4) + nokogiri (1.15.5) mini_portile2 (~> 2.8.2) racc (~> 1.4) - public_suffix (5.0.3) - racc (1.7.1) + public_suffix (5.0.4) + racc (1.7.3) rack (3.0.8) rack-test (2.1.0) rack (>= 1.3) @@ -45,7 +45,7 @@ GEM rspec-support (~> 3.12.0) rspec-support (3.12.1) rubyzip (2.3.2) - selenium-webdriver (4.14.0) + selenium-webdriver (4.15.0) rexml (~> 3.2, >= 3.2.5) rubyzip (>= 1.2.2, < 3.0) websocket (~> 1.0) From 49f2b6dcca688779793f72715e73a64d43629185 Mon Sep 17 00:00:00 2001 From: Jake Goulding Date: Tue, 21 Nov 2023 11:57:16 -0500 Subject: [PATCH 3/3] Discontinue building the separate tool containers --- .github/workflows/ci.yml | 57 +----------------------- .github/workflows/cron.yml | 52 ---------------------- ci/workflows.yml | 68 +---------------------------- compiler/build.sh | 17 -------- compiler/clippy/Dockerfile | 6 --- compiler/fetch.sh | 2 +- compiler/miri/Dockerfile | 6 --- compiler/miri/cargo-miri-playground | 7 --- compiler/rustfmt/Dockerfile | 6 --- 9 files changed, 4 insertions(+), 217 deletions(-) delete mode 100644 compiler/clippy/Dockerfile delete mode 100644 compiler/miri/Dockerfile delete mode 100755 compiler/miri/cargo-miri-playground delete mode 100644 compiler/rustfmt/Dockerfile diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e6885a7d5..63f851e83 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,44 +50,6 @@ jobs: tags: "${{ env.IMAGE_NAME }}:${{ github.run_id }}" cache-from: type=gha,scope=${{ matrix.channel }} cache-to: type=gha,scope=${{ matrix.channel }},mode=max - build_tool_containers: - name: Build ${{ matrix.tool }} tool container - runs-on: ubuntu-latest - needs: build_compiler_containers - strategy: - matrix: - tool: - - clippy - - miri - - rustfmt - if: 'github.event_name == ''push'' || contains(github.event.pull_request.labels.*.name, ''CI: approved'')' - env: - IMAGE_NAME: ghcr.io/integer32llc/rust-playground-ci-tool-${{ matrix.tool }} - steps: - - name: Checkout code - uses: actions/checkout@v3 - with: - ref: "${{ github.event.pull_request.head.sha }}" - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - with: - driver-opts: image=moby/buildkit:v0.11.6 - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: "${{ env.GH_CONTAINER_REGISTRY_USERNAME }}" - password: "${{ secrets.GH_CONTAINER_REGISTRY_TOKEN }}" - - name: Build and push container - uses: docker/build-push-action@v4 - with: - context: compiler/${{ matrix.tool }}/ - file: compiler/${{ matrix.tool }}/Dockerfile - build-args: base_image=ghcr.io/integer32llc/rust-playground-ci-rust-nightly:${{ github.run_id }} - push: true - tags: "${{ env.IMAGE_NAME }}:${{ github.run_id }}" - cache-from: type=gha,scope=${{ matrix.tool }} - cache-to: type=gha,scope=${{ matrix.tool }},mode=max build_backend: name: Build backend runs-on: ubuntu-latest @@ -171,7 +133,6 @@ jobs: if: 'github.event_name == ''push'' || contains(github.event.pull_request.labels.*.name, ''CI: approved'')' needs: - build_compiler_containers - - build_tool_containers - build_backend - build_frontend defaults: @@ -198,7 +159,7 @@ jobs: bundle config path vendor/bundle bundle install --jobs 4 --retry 3 - name: Pull containers - run: echo ghcr.io/integer32llc/rust-playground-ci-{rust-{stable,beta,nightly},tool-{clippy,rustfmt,miri}}:${{ github.run_id }} | xargs -n1 docker pull + run: echo ghcr.io/integer32llc/rust-playground-ci-rust-{stable,beta,nightly}:${{ github.run_id }} | xargs -n1 docker pull - name: Rename containers run: |- for c in stable beta nightly; do @@ -206,11 +167,6 @@ jobs: docker tag ghcr.io/integer32llc/rust-playground-ci-rust-$c:${{ github.run_id }} shepmaster/rust-$c docker tag ghcr.io/integer32llc/rust-playground-ci-rust-$c:${{ github.run_id }} rust-$c done - for t in clippy miri rustfmt; do - docker tag ghcr.io/integer32llc/rust-playground-ci-tool-$t:${{ github.run_id }} ghcr.io/integer32llc/rust-playground-ci-tool-$t - docker tag ghcr.io/integer32llc/rust-playground-ci-tool-$t:${{ github.run_id }} shepmaster/$t - docker tag ghcr.io/integer32llc/rust-playground-ci-tool-$t:${{ github.run_id }} $t - done - name: Download backend uses: actions/download-artifact@v3 with: @@ -265,7 +221,7 @@ jobs: username: "${{ env.DOCKER_HUB_USERNAME }}" password: "${{ secrets.DOCKER_HUB_TOKEN }}" - name: Pull containers - run: echo ghcr.io/integer32llc/rust-playground-ci-{rust-{stable,beta,nightly},tool-{clippy,rustfmt,miri}}:${{ github.run_id }} | xargs -n1 docker pull + run: echo ghcr.io/integer32llc/rust-playground-ci-rust-{stable,beta,nightly}:${{ github.run_id }} | xargs -n1 docker pull - name: Rename containers run: |- for c in stable beta nightly; do @@ -273,21 +229,12 @@ jobs: docker tag ghcr.io/integer32llc/rust-playground-ci-rust-$c:${{ github.run_id }} shepmaster/rust-$c docker tag ghcr.io/integer32llc/rust-playground-ci-rust-$c:${{ github.run_id }} rust-$c done - for t in clippy miri rustfmt; do - docker tag ghcr.io/integer32llc/rust-playground-ci-tool-$t:${{ github.run_id }} ghcr.io/integer32llc/rust-playground-ci-tool-$t - docker tag ghcr.io/integer32llc/rust-playground-ci-tool-$t:${{ github.run_id }} shepmaster/$t - docker tag ghcr.io/integer32llc/rust-playground-ci-tool-$t:${{ github.run_id }} $t - done - name: Push containers run: |- for c in stable beta nightly; do docker push ghcr.io/integer32llc/rust-playground-ci-rust-$c docker push shepmaster/rust-$c done - for t in clippy miri rustfmt; do - docker push ghcr.io/integer32llc/rust-playground-ci-tool-$t - docker push shepmaster/$t - done - name: Download backend uses: actions/download-artifact@v3 with: diff --git a/.github/workflows/cron.yml b/.github/workflows/cron.yml index 4db275175..d7086da0c 100644 --- a/.github/workflows/cron.yml +++ b/.github/workflows/cron.yml @@ -61,55 +61,3 @@ jobs: run: |- docker push ${{ env.IMAGE_NAME }} docker push ${{ env.DOCKER_HUB_IMAGE_NAME }} - build_tool_containers: - name: Build ${{ matrix.tool }} tool container - runs-on: ubuntu-latest - needs: build_compiler_containers - strategy: - matrix: - tool: - - clippy - - miri - - rustfmt - env: - IMAGE_NAME: ghcr.io/integer32llc/rust-playground-ci-tool-${{ matrix.tool }} - DOCKER_HUB_IMAGE_NAME: shepmaster/${{ matrix.tool }} - continue-on-error: true - steps: - - name: Checkout code - uses: actions/checkout@v3 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - with: - driver-opts: image=moby/buildkit:v0.11.6 - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: "${{ env.GH_CONTAINER_REGISTRY_USERNAME }}" - password: "${{ secrets.GH_CONTAINER_REGISTRY_TOKEN }}" - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: "${{ env.DOCKER_HUB_USERNAME }}" - password: "${{ secrets.DOCKER_HUB_TOKEN }}" - - name: Build and push container - uses: docker/build-push-action@v4 - with: - context: compiler/${{ matrix.tool }}/ - file: compiler/${{ matrix.tool }}/Dockerfile - build-args: base_image=ghcr.io/integer32llc/rust-playground-ci-rust-nightly:${{ github.run_id }} - push: true - tags: "${{ env.IMAGE_NAME }}:${{ github.run_id }}" - cache-from: type=gha,scope=${{ matrix.tool }} - cache-to: type=gha,scope=${{ matrix.tool }},mode=max - - name: Pull container - run: docker pull ${{ env.IMAGE_NAME }}:${{ github.run_id }} - - name: Rename container - run: |- - docker tag ${{ env.IMAGE_NAME }}:${{ github.run_id }} ${{ env.IMAGE_NAME }} - docker tag ${{ env.IMAGE_NAME }}:${{ github.run_id }} ${{ env.DOCKER_HUB_IMAGE_NAME }} - - name: Push container - run: |- - docker push ${{ env.IMAGE_NAME }} - docker push ${{ env.DOCKER_HUB_IMAGE_NAME }} diff --git a/ci/workflows.yml b/ci/workflows.yml index 51b387b86..683468067 100644 --- a/ci/workflows.yml +++ b/ci/workflows.yml @@ -64,36 +64,10 @@ components: cache-from: type=gha,scope=${{ matrix.channel }} cache-to: type=gha,scope=${{ matrix.channel }},mode=max - - build_tool_containers_job: &build_tool_containers_job - name: "Build ${{ matrix.tool }} tool container" - runs-on: ubuntu-latest - needs: build_compiler_containers - - strategy: - matrix: - tool: [clippy, miri, rustfmt] - - - build_tool_containers_job_env: &build_tool_containers_job_env - IMAGE_NAME: ghcr.io/integer32llc/rust-playground-ci-tool-${{ matrix.tool }} - - - build_tool_containers: &build_tool_containers - name: "Build and push container" - uses: docker/build-push-action@v4 - with: - context: compiler/${{ matrix.tool }}/ - file: compiler/${{ matrix.tool }}/Dockerfile - build-args: |- - base_image=ghcr.io/integer32llc/rust-playground-ci-rust-nightly:${{ github.run_id }} - push: true - tags: |- - ${{ env.IMAGE_NAME }}:${{ github.run_id }} - cache-from: type=gha,scope=${{ matrix.tool }} - cache-to: type=gha,scope=${{ matrix.tool }},mode=max - - pull_containers: &pull_containers name: "Pull containers" run: |- - echo ghcr.io/integer32llc/rust-playground-ci-{rust-{stable,beta,nightly},tool-{clippy,rustfmt,miri}}:${{ github.run_id }} | xargs -n1 docker pull + echo ghcr.io/integer32llc/rust-playground-ci-rust-{stable,beta,nightly}:${{ github.run_id }} | xargs -n1 docker pull - rename_all_containers: &rename_all_containers name: "Rename containers" @@ -103,11 +77,6 @@ components: docker tag ghcr.io/integer32llc/rust-playground-ci-rust-$c:${{ github.run_id }} shepmaster/rust-$c docker tag ghcr.io/integer32llc/rust-playground-ci-rust-$c:${{ github.run_id }} rust-$c done - for t in clippy miri rustfmt; do - docker tag ghcr.io/integer32llc/rust-playground-ci-tool-$t:${{ github.run_id }} ghcr.io/integer32llc/rust-playground-ci-tool-$t - docker tag ghcr.io/integer32llc/rust-playground-ci-tool-$t:${{ github.run_id }} shepmaster/$t - docker tag ghcr.io/integer32llc/rust-playground-ci-tool-$t:${{ github.run_id }} $t - done - pull_current_container: &pull_current_container name: "Pull container" @@ -152,18 +121,6 @@ workflows: - *login_ghcr - *build_compiler_containers - build_tool_containers: - <<: *build_tool_containers_job - if: "github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'CI: approved')" - env: - <<: *build_tool_containers_job_env - - steps: - - *checkout_pr - - *docker_buildx - - *login_ghcr - - *build_tool_containers - build_backend: name: "Build backend" runs-on: ubuntu-latest @@ -266,7 +223,6 @@ workflows: if: "github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'CI: approved')" needs: - build_compiler_containers - - build_tool_containers - build_backend - build_frontend @@ -367,10 +323,6 @@ workflows: docker push ghcr.io/integer32llc/rust-playground-ci-rust-$c docker push shepmaster/rust-$c done - for t in clippy miri rustfmt; do - docker push ghcr.io/integer32llc/rust-playground-ci-tool-$t - docker push shepmaster/$t - done - name: "Download backend" uses: actions/download-artifact@v3 @@ -444,21 +396,3 @@ workflows: - *pull_current_container - *rename_current_container - *push_current_container - - build_tool_containers: - <<: *build_tool_containers_job - env: - <<: *build_tool_containers_job_env - DOCKER_HUB_IMAGE_NAME: shepmaster/${{ matrix.tool }} - continue-on-error: true - - steps: - - *checkout - - *docker_buildx - - *login_ghcr - - *login_docker_hub - - *build_tool_containers - - - *pull_current_container - - *rename_current_container - - *push_current_container diff --git a/compiler/build.sh b/compiler/build.sh index a594a81bb..fd24f8836 100755 --- a/compiler/build.sh +++ b/compiler/build.sh @@ -3,7 +3,6 @@ set -euv -o pipefail channels_to_build="${CHANNELS_TO_BUILD-stable beta nightly}" -tools_to_build="${TOOLS_TO_BUILD-rustfmt clippy miri}" repository=shepmaster @@ -21,19 +20,3 @@ for channel in $channels_to_build; do cd .. done - -crate_api_base=https://crates.io/api/v1/crates - -for tool in $tools_to_build; do - cd "${tool}" - - image_name="${tool}" - full_name="${repository}/${image_name}" - - docker build -t "${full_name}" \ - . - - docker tag "${full_name}" "${image_name}" - - cd .. -done diff --git a/compiler/clippy/Dockerfile b/compiler/clippy/Dockerfile deleted file mode 100644 index aa8e1afb1..000000000 --- a/compiler/clippy/Dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -ARG base_image=shepmaster/rust-nightly -FROM ${base_image} - -# The base image takes care of all this for now - -ENTRYPOINT ["/playground/tools/entrypoint.sh"] diff --git a/compiler/fetch.sh b/compiler/fetch.sh index 12539a7c7..ce455ff64 100755 --- a/compiler/fetch.sh +++ b/compiler/fetch.sh @@ -4,7 +4,7 @@ set -euv -o pipefail repository=shepmaster -for image in rust-stable rust-beta rust-nightly rustfmt clippy miri; do +for image in rust-stable rust-beta rust-nightly; do docker pull "${repository}/${image}" # The backend expects images without a repository prefix docker tag "${repository}/${image}" "${image}" diff --git a/compiler/miri/Dockerfile b/compiler/miri/Dockerfile deleted file mode 100644 index aa8e1afb1..000000000 --- a/compiler/miri/Dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -ARG base_image=shepmaster/rust-nightly -FROM ${base_image} - -# The base image takes care of all this for now - -ENTRYPOINT ["/playground/tools/entrypoint.sh"] diff --git a/compiler/miri/cargo-miri-playground b/compiler/miri/cargo-miri-playground deleted file mode 100755 index ae6e37408..000000000 --- a/compiler/miri/cargo-miri-playground +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash - -set -eu - -export MIRI_SYSROOT=~/.cache/miri -export MIRIFLAGS="-Zmiri-disable-isolation" -exec cargo miri run diff --git a/compiler/rustfmt/Dockerfile b/compiler/rustfmt/Dockerfile deleted file mode 100644 index aa8e1afb1..000000000 --- a/compiler/rustfmt/Dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -ARG base_image=shepmaster/rust-nightly -FROM ${base_image} - -# The base image takes care of all this for now - -ENTRYPOINT ["/playground/tools/entrypoint.sh"]