From 34f60c200442a7a1c8b70c1d17f0eedf92f46c0c Mon Sep 17 00:00:00 2001 From: Jake Goulding <jake.goulding@integer32.com> Date: Tue, 21 Nov 2023 13:19:12 -0500 Subject: [PATCH 1/3] Migrate metrics to coordinator types --- ui/src/metrics.rs | 79 ++++++++++++++++++++++++-------------- ui/src/sandbox.rs | 97 ----------------------------------------------- 2 files changed, 51 insertions(+), 125 deletions(-) diff --git a/ui/src/metrics.rs b/ui/src/metrics.rs index c9c970512..214c39b12 100644 --- a/ui/src/metrics.rs +++ b/ui/src/metrics.rs @@ -1,5 +1,5 @@ use lazy_static::lazy_static; -use orchestrator::coordinator; +use orchestrator::coordinator::{self, Channel, CompileTarget, CrateType, Edition, Mode}; use prometheus::{ self, register_histogram, register_histogram_vec, register_int_counter, register_int_counter_vec, register_int_gauge, Histogram, HistogramVec, IntCounter, @@ -10,7 +10,7 @@ use std::{ time::{Duration, Instant}, }; -use crate::sandbox::{self, Channel, CompileTarget, CrateType, Edition, Mode}; +use crate::sandbox; lazy_static! { pub(crate) static ref REQUESTS: HistogramVec = register_histogram_vec!( @@ -132,15 +132,38 @@ impl Labels { v.map_or("", |v| if v { "true" } else { "false" }) } - let target = target.map_or("", Into::into); - let channel = channel.map_or("", Into::into); - let mode = mode.map_or("", Into::into); + let target = match target { + Some(CompileTarget::Assembly(_, _, _)) => "Assembly", + Some(CompileTarget::Hir) => "Hir", + Some(CompileTarget::LlvmIr) => "LlvmIr", + Some(CompileTarget::Mir) => "Mir", + Some(CompileTarget::Wasm) => "Wasm", + None => "", + }; + let channel = match channel { + Some(Channel::Stable) => "Stable", + Some(Channel::Beta) => "Beta", + Some(Channel::Nightly) => "Nightly", + None => "", + }; + let mode = match mode { + Some(Mode::Debug) => "Debug", + Some(Mode::Release) => "Release", + None => "", + }; let edition = match edition { None => "", Some(None) => "Unspecified", - Some(Some(v)) => v.into(), + Some(Some(Edition::Rust2015)) => "Rust2015", + Some(Some(Edition::Rust2018)) => "Rust2018", + Some(Some(Edition::Rust2021)) => "Rust2021", + Some(Some(Edition::Rust2024)) => "Rust2024", + }; + let crate_type = match crate_type { + Some(CrateType::Binary) => "Binary", + Some(CrateType::Library(_)) => "Library", + None => "", }; - let crate_type = crate_type.map_or("", Into::into); let tests = b(tests); let backtrace = b(backtrace); @@ -262,11 +285,11 @@ impl HasLabelsCore for coordinator::CompileRequest { } = *self; LabelsCore { - target: Some(target.into()), - channel: Some(channel.into()), - mode: Some(mode.into()), - edition: Some(Some(edition.into())), - crate_type: Some(crate_type.into()), + target: Some(target), + channel: Some(channel), + mode: Some(mode), + edition: Some(Some(edition)), + crate_type: Some(crate_type), tests: Some(tests), backtrace: Some(backtrace), } @@ -287,10 +310,10 @@ impl HasLabelsCore for coordinator::ExecuteRequest { LabelsCore { target: None, - channel: Some(channel.into()), - mode: Some(mode.into()), - edition: Some(Some(edition.into())), - crate_type: Some(crate_type.into()), + channel: Some(channel), + mode: Some(mode), + edition: Some(Some(edition)), + crate_type: Some(crate_type), tests: Some(tests), backtrace: Some(backtrace), } @@ -308,10 +331,10 @@ impl HasLabelsCore for coordinator::FormatRequest { LabelsCore { target: None, - channel: Some(channel.into()), + channel: Some(channel), mode: None, - edition: Some(Some(edition.into())), - crate_type: Some(crate_type.into()), + edition: Some(Some(edition)), + crate_type: Some(crate_type), tests: None, backtrace: None, } @@ -329,10 +352,10 @@ impl HasLabelsCore for coordinator::ClippyRequest { LabelsCore { target: None, - channel: Some(channel.into()), + channel: Some(channel), mode: None, - edition: Some(Some(edition.into())), - crate_type: Some(crate_type.into()), + edition: Some(Some(edition)), + crate_type: Some(crate_type), tests: None, backtrace: None, } @@ -350,10 +373,10 @@ impl HasLabelsCore for coordinator::MiriRequest { LabelsCore { target: None, - channel: Some(channel.into()), + channel: Some(channel), mode: None, - edition: Some(Some(edition.into())), - crate_type: Some(crate_type.into()), + edition: Some(Some(edition)), + crate_type: Some(crate_type), tests: None, backtrace: None, } @@ -371,10 +394,10 @@ impl HasLabelsCore for coordinator::MacroExpansionRequest { LabelsCore { target: None, - channel: Some(channel.into()), + channel: Some(channel), mode: None, - edition: Some(Some(edition.into())), - crate_type: Some(crate_type.into()), + edition: Some(Some(edition)), + crate_type: Some(crate_type), tests: None, backtrace: None, } diff --git a/ui/src/sandbox.rs b/ui/src/sandbox.rs index ae1a3b605..99939f19a 100644 --- a/ui/src/sandbox.rs +++ b/ui/src/sandbox.rs @@ -473,100 +473,3 @@ impl<R: BacktraceRequest> BacktraceRequest for &'_ R { (*self).backtrace() } } - -mod sandbox_orchestrator_integration_impls { - use orchestrator::coordinator; - - impl From<coordinator::CompileTarget> for super::CompileTarget { - fn from(value: coordinator::CompileTarget) -> Self { - match value { - coordinator::CompileTarget::Assembly(a, b, c) => { - super::CompileTarget::Assembly(a.into(), b.into(), c.into()) - } - coordinator::CompileTarget::Hir => super::CompileTarget::Hir, - coordinator::CompileTarget::LlvmIr => super::CompileTarget::LlvmIr, - coordinator::CompileTarget::Mir => super::CompileTarget::Mir, - coordinator::CompileTarget::Wasm => super::CompileTarget::Wasm, - } - } - } - - impl From<coordinator::Mode> for super::Mode { - fn from(value: coordinator::Mode) -> Self { - match value { - coordinator::Mode::Debug => super::Mode::Debug, - coordinator::Mode::Release => super::Mode::Release, - } - } - } - - impl From<coordinator::Edition> for super::Edition { - fn from(value: coordinator::Edition) -> Self { - match value { - coordinator::Edition::Rust2015 => super::Edition::Rust2015, - coordinator::Edition::Rust2018 => super::Edition::Rust2018, - coordinator::Edition::Rust2021 => super::Edition::Rust2021, - coordinator::Edition::Rust2024 => super::Edition::Rust2024, - } - } - } - - impl From<coordinator::Channel> for super::Channel { - fn from(value: coordinator::Channel) -> Self { - match value { - coordinator::Channel::Stable => super::Channel::Stable, - coordinator::Channel::Beta => super::Channel::Beta, - coordinator::Channel::Nightly => super::Channel::Nightly, - } - } - } - - impl From<coordinator::AssemblyFlavor> for super::AssemblyFlavor { - fn from(value: coordinator::AssemblyFlavor) -> Self { - match value { - coordinator::AssemblyFlavor::Att => super::AssemblyFlavor::Att, - coordinator::AssemblyFlavor::Intel => super::AssemblyFlavor::Intel, - } - } - } - - impl From<coordinator::CrateType> for super::CrateType { - fn from(value: coordinator::CrateType) -> Self { - match value { - coordinator::CrateType::Binary => super::CrateType::Binary, - coordinator::CrateType::Library(a) => super::CrateType::Library(a.into()), - } - } - } - - impl From<coordinator::DemangleAssembly> for super::DemangleAssembly { - fn from(value: coordinator::DemangleAssembly) -> Self { - match value { - coordinator::DemangleAssembly::Demangle => super::DemangleAssembly::Demangle, - coordinator::DemangleAssembly::Mangle => super::DemangleAssembly::Mangle, - } - } - } - - impl From<coordinator::ProcessAssembly> for super::ProcessAssembly { - fn from(value: coordinator::ProcessAssembly) -> Self { - match value { - coordinator::ProcessAssembly::Filter => super::ProcessAssembly::Filter, - coordinator::ProcessAssembly::Raw => super::ProcessAssembly::Raw, - } - } - } - - impl From<coordinator::LibraryType> for super::LibraryType { - fn from(value: coordinator::LibraryType) -> Self { - match value { - coordinator::LibraryType::Lib => super::LibraryType::Lib, - coordinator::LibraryType::Dylib => super::LibraryType::Dylib, - coordinator::LibraryType::Rlib => super::LibraryType::Rlib, - coordinator::LibraryType::Staticlib => super::LibraryType::Staticlib, - coordinator::LibraryType::Cdylib => super::LibraryType::Cdylib, - coordinator::LibraryType::ProcMacro => super::LibraryType::ProcMacro, - } - } - } -} From 9f4880c33b0744ef8478b8d5422276d53d23c75b Mon Sep 17 00:00:00 2001 From: Jake Goulding <jake.goulding@integer32.com> Date: Tue, 21 Nov 2023 13:22:00 -0500 Subject: [PATCH 2/3] Remove newly-dead code --- ui/src/sandbox.rs | 167 +--------------------------------------------- 1 file changed, 1 insertion(+), 166 deletions(-) diff --git a/ui/src/sandbox.rs b/ui/src/sandbox.rs index 99939f19a..d7804e068 100644 --- a/ui/src/sandbox.rs +++ b/ui/src/sandbox.rs @@ -1,6 +1,6 @@ use serde_derive::Deserialize; use snafu::prelude::*; -use std::{collections::BTreeMap, fmt, io, string, time::Duration}; +use std::{collections::BTreeMap, io, string, time::Duration}; use tempfile::TempDir; use tokio::{process::Command, time}; @@ -290,47 +290,6 @@ async fn run_command_with_timeout(mut command: Command) -> Result<std::process:: Ok(output) } -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -pub enum AssemblyFlavor { - Att, - Intel, -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -pub enum DemangleAssembly { - Demangle, - Mangle, -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -pub enum ProcessAssembly { - Filter, - Raw, -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, strum::IntoStaticStr)] -pub enum CompileTarget { - Assembly(AssemblyFlavor, DemangleAssembly, ProcessAssembly), - LlvmIr, - Mir, - Hir, - Wasm, -} - -impl fmt::Display for CompileTarget { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - use self::CompileTarget::*; - - match *self { - Assembly(_, _, _) => "assembly".fmt(f), - LlvmIr => "LLVM IR".fmt(f), - Mir => "Rust MIR".fmt(f), - Hir => "Rust HIR".fmt(f), - Wasm => "WebAssembly".fmt(f), - } - } -} - #[derive(Debug, Copy, Clone, PartialEq, Eq, strum::IntoStaticStr)] pub enum Channel { Stable, @@ -349,127 +308,3 @@ impl Channel { } } } - -#[derive(Debug, Copy, Clone, PartialEq, Eq, strum::IntoStaticStr)] -pub enum Mode { - Debug, - Release, -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, strum::IntoStaticStr)] -pub enum Edition { - Rust2015, - Rust2018, - Rust2021, // TODO - add parallel tests for 2021 - Rust2024, -} - -impl Edition { - fn cargo_ident(&self) -> &'static str { - use self::Edition::*; - - match *self { - Rust2015 => "2015", - Rust2018 => "2018", - Rust2021 => "2021", - Rust2024 => "2024", - } - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, strum::IntoStaticStr)] -pub enum CrateType { - Binary, - Library(LibraryType), -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, strum::IntoStaticStr)] -pub enum LibraryType { - Lib, - Dylib, - Rlib, - Staticlib, - Cdylib, - ProcMacro, -} - -impl LibraryType { - fn cargo_ident(&self) -> &'static str { - use self::LibraryType::*; - - match *self { - Lib => "lib", - Dylib => "dylib", - Rlib => "rlib", - Staticlib => "staticlib", - Cdylib => "cdylib", - ProcMacro => "proc-macro", - } - } -} - -trait DockerCommandExt { - fn apply_crate_type(&mut self, req: impl CrateTypeRequest); - fn apply_edition(&mut self, req: impl EditionRequest); - fn apply_backtrace(&mut self, req: impl BacktraceRequest); -} - -impl DockerCommandExt for Command { - fn apply_crate_type(&mut self, req: impl CrateTypeRequest) { - if let CrateType::Library(lib) = req.crate_type() { - self.args(&[ - "--env", - &format!("PLAYGROUND_CRATE_TYPE={}", lib.cargo_ident()), - ]); - } - } - - fn apply_edition(&mut self, req: impl EditionRequest) { - if let Some(edition) = req.edition() { - if edition == Edition::Rust2024 { - self.args(&["--env", &format!("PLAYGROUND_FEATURE_EDITION2024=true")]); - } - - self.args(&[ - "--env", - &format!("PLAYGROUND_EDITION={}", edition.cargo_ident()), - ]); - } - } - - fn apply_backtrace(&mut self, req: impl BacktraceRequest) { - if req.backtrace() { - self.args(&["--env", "RUST_BACKTRACE=1"]); - } - } -} - -trait CrateTypeRequest { - fn crate_type(&self) -> CrateType; -} - -impl<R: CrateTypeRequest> CrateTypeRequest for &'_ R { - fn crate_type(&self) -> CrateType { - (*self).crate_type() - } -} - -trait EditionRequest { - fn edition(&self) -> Option<Edition>; -} - -impl<R: EditionRequest> EditionRequest for &'_ R { - fn edition(&self) -> Option<Edition> { - (*self).edition() - } -} - -trait BacktraceRequest { - fn backtrace(&self) -> bool; -} - -impl<R: BacktraceRequest> BacktraceRequest for &'_ R { - fn backtrace(&self) -> bool { - (*self).backtrace() - } -} From 235ffe9f0064290160afb9c6164be40be8d1f306 Mon Sep 17 00:00:00 2001 From: Jake Goulding <jake.goulding@integer32.com> Date: Wed, 22 Nov 2023 14:33:20 -0500 Subject: [PATCH 3/3] Migrate version reporting to the orchestrator --- compiler/base/orchestrator/src/coordinator.rs | 230 ++++++++++++++++++ compiler/base/orchestrator/src/message.rs | 14 ++ ui/src/main.rs | 18 +- ui/src/metrics.rs | 6 - ui/src/sandbox.rs | 91 +------ ui/src/server_axum.rs | 150 ++++++++---- 6 files changed, 351 insertions(+), 158 deletions(-) diff --git a/compiler/base/orchestrator/src/coordinator.rs b/compiler/base/orchestrator/src/coordinator.rs index 2d58a0c10..2e4cf73f1 100644 --- a/compiler/base/orchestrator/src/coordinator.rs +++ b/compiler/base/orchestrator/src/coordinator.rs @@ -35,6 +35,134 @@ use crate::{ DropErrorDetailsExt, }; +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Versions { + pub stable: ChannelVersions, + pub beta: ChannelVersions, + pub nightly: ChannelVersions, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ChannelVersions { + pub rustc: Version, + pub rustfmt: Version, + pub clippy: Version, + pub miri: Option<Version>, +} + +/// Parsing this struct is very lenient — we'd rather return some +/// partial data instead of absolutely nothing. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Version { + pub release: String, + pub commit_hash: String, + pub commit_date: String, +} + +impl Version { + fn parse_rustc_version_verbose(rustc_version: &str) -> Self { + let mut release = ""; + let mut commit_hash = ""; + let mut commit_date = ""; + + let fields = rustc_version.lines().skip(1).filter_map(|line| { + let mut pieces = line.splitn(2, ':'); + let key = pieces.next()?.trim(); + let value = pieces.next()?.trim(); + Some((key, value)) + }); + + for (k, v) in fields { + match k { + "release" => release = v, + "commit-hash" => commit_hash = v, + "commit-date" => commit_date = v, + _ => {} + } + } + + Self { + release: release.into(), + commit_hash: commit_hash.into(), + commit_date: commit_date.into(), + } + } + + // Parses versions of the shape `toolname 0.0.0 (0000000 0000-00-00)` + fn parse_tool_version(tool_version: &str) -> Self { + let mut parts = tool_version.split_whitespace().fuse().skip(1); + + let release = parts.next().unwrap_or("").into(); + let commit_hash = parts.next().unwrap_or("").trim_start_matches('(').into(); + let commit_date = parts.next().unwrap_or("").trim_end_matches(')').into(); + + Self { + release, + commit_hash, + commit_date, + } + } +} + +#[derive(Debug, Snafu)] +#[snafu(module)] +pub enum VersionsError { + #[snafu(display("Unable to determine versions for the stable channel"))] + Stable { source: VersionsChannelError }, + + #[snafu(display("Unable to determine versions for the beta channel"))] + Beta { source: VersionsChannelError }, + + #[snafu(display("Unable to determine versions for the nightly channel"))] + Nightly { source: VersionsChannelError }, +} + +#[derive(Debug, Snafu)] +pub enum VersionsChannelError { + #[snafu(context(false))] // transparent + Channel { source: Error }, + + #[snafu(context(false))] // transparent + Versions { source: ContainerVersionsError }, +} + +#[derive(Debug, Snafu)] +#[snafu(module)] +pub enum ContainerVersionsError { + #[snafu(display("Failed to get `rustc` version"))] + Rustc { source: VersionError }, + + #[snafu(display("`rustc` not executable"))] + RustcMissing, + + #[snafu(display("Failed to get `rustfmt` version"))] + Rustfmt { source: VersionError }, + + #[snafu(display("`cargo fmt` not executable"))] + RustfmtMissing, + + #[snafu(display("Failed to get clippy version"))] + Clippy { source: VersionError }, + + #[snafu(display("`cargo clippy` not executable"))] + ClippyMissing, + + #[snafu(display("Failed to get miri version"))] + Miri { source: VersionError }, +} + +#[derive(Debug, Snafu)] +#[snafu(module)] +pub enum VersionError { + #[snafu(display("Could not start the process"))] + #[snafu(context(false))] + SpawnProcess { source: SpawnCargoError }, + + #[snafu(display("The task panicked"))] + #[snafu(context(false))] + TaskPanic { source: tokio::task::JoinError }, +} + #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub enum AssemblyFlavor { Att, @@ -639,6 +767,28 @@ where } } + pub async fn versions(&self) -> Result<Versions, VersionsError> { + use versions_error::*; + + let [stable, beta, nightly] = + [Channel::Stable, Channel::Beta, Channel::Nightly].map(|c| async move { + let c = self.select_channel(c).await?; + c.versions().await.map_err(VersionsChannelError::from) + }); + + let (stable, beta, nightly) = join!(stable, beta, nightly); + + let stable = stable.context(StableSnafu)?; + let beta = beta.context(BetaSnafu)?; + let nightly = nightly.context(NightlySnafu)?; + + Ok(Versions { + stable, + beta, + nightly, + }) + } + pub async fn execute( &self, request: ExecuteRequest, @@ -904,6 +1054,72 @@ impl Container { }) } + async fn versions(&self) -> Result<ChannelVersions, ContainerVersionsError> { + use container_versions_error::*; + + let token = CancellationToken::new(); + + let rustc = self.rustc_version(token.clone()); + let rustfmt = self.tool_version(token.clone(), "fmt"); + let clippy = self.tool_version(token.clone(), "clippy"); + let miri = self.tool_version(token, "miri"); + + let (rustc, rustfmt, clippy, miri) = join!(rustc, rustfmt, clippy, miri); + + let rustc = rustc.context(RustcSnafu)?.context(RustcMissingSnafu)?; + let rustfmt = rustfmt + .context(RustfmtSnafu)? + .context(RustfmtMissingSnafu)?; + let clippy = clippy.context(ClippySnafu)?.context(ClippyMissingSnafu)?; + let miri = miri.context(MiriSnafu)?; + + Ok(ChannelVersions { + rustc, + rustfmt, + clippy, + miri, + }) + } + + async fn rustc_version( + &self, + token: CancellationToken, + ) -> Result<Option<Version>, VersionError> { + let rustc_cmd = ExecuteCommandRequest::simple("rustc", ["--version", "--verbose"]); + let output = self.version_output(token, rustc_cmd).await?; + + Ok(output.map(|o| Version::parse_rustc_version_verbose(&o))) + } + + async fn tool_version( + &self, + token: CancellationToken, + subcommand_name: &str, + ) -> Result<Option<Version>, VersionError> { + let tool_cmd = ExecuteCommandRequest::simple("cargo", [subcommand_name, "--version"]); + let output = self.version_output(token, tool_cmd).await?; + + Ok(output.map(|o| Version::parse_tool_version(&o))) + } + + async fn version_output( + &self, + token: CancellationToken, + cmd: ExecuteCommandRequest, + ) -> Result<Option<String>, VersionError> { + let v = self.spawn_cargo_task(token.clone(), cmd).await?; + let SpawnCargo { + task, + stdin_tx, + stdout_rx, + stderr_rx, + } = v; + drop(stdin_tx); + let task = async { task.await?.map_err(VersionError::from) }; + let o = WithOutput::try_absorb(task, stdout_rx, stderr_rx).await?; + Ok(if o.success { Some(o.stdout) } else { None }) + } + async fn execute( &self, request: ExecuteRequest, @@ -2416,6 +2632,20 @@ mod tests { } } + #[tokio::test] + #[snafu::report] + async fn versions() -> Result<()> { + let coordinator = new_coordinator().await; + + let versions = coordinator.versions().with_timeout().await.unwrap(); + + assert_starts_with!(versions.stable.rustc.release, "1."); + + coordinator.shutdown().await?; + + Ok(()) + } + const ARBITRARY_EXECUTE_REQUEST: ExecuteRequest = ExecuteRequest { channel: Channel::Stable, mode: Mode::Debug, diff --git a/compiler/base/orchestrator/src/message.rs b/compiler/base/orchestrator/src/message.rs index 36fd2227c..fe728d5b4 100644 --- a/compiler/base/orchestrator/src/message.rs +++ b/compiler/base/orchestrator/src/message.rs @@ -116,6 +116,20 @@ pub struct ExecuteCommandRequest { pub cwd: Option<String>, // None means in project direcotry. } +impl ExecuteCommandRequest { + pub fn simple( + cmd: impl Into<String>, + args: impl IntoIterator<Item = impl Into<String>>, + ) -> Self { + Self { + cmd: cmd.into(), + args: args.into_iter().map(Into::into).collect(), + envs: Default::default(), + cwd: None, + } + } +} + #[derive(Debug, Serialize, Deserialize)] pub struct ExecuteCommandResponse { pub success: bool, diff --git a/ui/src/main.rs b/ui/src/main.rs index 51f988061..1fbcf1d52 100644 --- a/ui/src/main.rs +++ b/ui/src/main.rs @@ -219,6 +219,14 @@ enum Error { #[snafu(display("The WebSocket worker panicked: {}", text))] WebSocketTaskPanic { text: String }, + #[snafu(display("Unable to find the available versions"))] + Versions { + source: orchestrator::coordinator::VersionsError, + }, + + #[snafu(display("The Miri version was missing"))] + MiriVersion, + #[snafu(display("Unable to shutdown the coordinator"))] ShutdownCoordinator { source: orchestrator::coordinator::Error, @@ -471,16 +479,6 @@ impl From<Vec<sandbox::CrateInformation>> for MetaCratesResponse { } } -impl From<sandbox::Version> for MetaVersionResponse { - fn from(me: sandbox::Version) -> Self { - MetaVersionResponse { - version: me.release.into(), - hash: me.commit_hash.into(), - date: me.commit_date.into(), - } - } -} - impl From<gist::Gist> for MetaGistResponse { fn from(me: gist::Gist) -> Self { MetaGistResponse { diff --git a/ui/src/metrics.rs b/ui/src/metrics.rs index 214c39b12..e1da4f64c 100644 --- a/ui/src/metrics.rs +++ b/ui/src/metrics.rs @@ -227,12 +227,6 @@ impl SuccessDetails for Vec<sandbox::CrateInformation> { } } -impl SuccessDetails for sandbox::Version { - fn success_details(&self) -> Outcome { - Outcome::Success - } -} - pub(crate) async fn track_metric_no_request_async<B, Fut, Resp>( endpoint: Endpoint, body: B, diff --git a/ui/src/sandbox.rs b/ui/src/sandbox.rs index d7804e068..27539aa85 100644 --- a/ui/src/sandbox.rs +++ b/ui/src/sandbox.rs @@ -1,6 +1,6 @@ use serde_derive::Deserialize; use snafu::prelude::*; -use std::{collections::BTreeMap, io, string, time::Duration}; +use std::{io, time::Duration}; use tempfile::TempDir; use tokio::{process::Command, time}; @@ -28,13 +28,6 @@ impl From<CrateInformationInner> for CrateInformation { } } -#[derive(Debug, Clone)] -pub struct Version { - pub release: String, - pub commit_hash: String, - pub commit_date: String, -} - #[derive(Debug, Snafu)] pub enum Error { #[snafu(display("Unable to create temporary directory: {}", source))] @@ -58,22 +51,10 @@ pub enum Error { #[snafu(display("Unable to read crate information: {}", source))] UnableToParseCrateInformation { source: ::serde_json::Error }, - #[snafu(display("Output was not valid UTF-8: {}", source))] - OutputNotUtf8 { source: string::FromUtf8Error }, - #[snafu(display("Release was missing from the version output"))] - VersionReleaseMissing, - #[snafu(display("Commit hash was missing from the version output"))] - VersionHashMissing, - #[snafu(display("Commit date was missing from the version output"))] - VersionDateMissing, } pub type Result<T, E = Error> = ::std::result::Result<T, E>; -fn vec_to_str(v: Vec<u8>) -> Result<String> { - String::from_utf8(v).context(OutputNotUtf8Snafu) -} - macro_rules! docker_command { ($($arg:expr),* $(,)?) => ({ let mut cmd = Command::new("docker"); @@ -151,76 +132,6 @@ impl Sandbox { Ok(crates) } - - pub async fn version(&self, channel: Channel) -> Result<Version> { - let mut command = basic_secure_docker_command(); - command.args(&[channel.container_name()]); - command.args(&["rustc", "--version", "--verbose"]); - - let output = run_command_with_timeout(command).await?; - let version_output = vec_to_str(output.stdout)?; - - let mut info: BTreeMap<String, String> = version_output - .lines() - .skip(1) - .filter_map(|line| { - let mut pieces = line.splitn(2, ':').fuse(); - match (pieces.next(), pieces.next()) { - (Some(name), Some(value)) => Some((name.trim().into(), value.trim().into())), - _ => None, - } - }) - .collect(); - - let release = info.remove("release").context(VersionReleaseMissingSnafu)?; - let commit_hash = info - .remove("commit-hash") - .context(VersionHashMissingSnafu)?; - let commit_date = info - .remove("commit-date") - .context(VersionDateMissingSnafu)?; - - Ok(Version { - release, - commit_hash, - commit_date, - }) - } - - pub async fn version_rustfmt(&self) -> Result<Version> { - let mut command = basic_secure_docker_command(); - command.args(&["rustfmt", "cargo", "fmt", "--version"]); - self.cargo_tool_version(command).await - } - - pub async fn version_clippy(&self) -> Result<Version> { - let mut command = basic_secure_docker_command(); - command.args(&["clippy", "cargo", "clippy", "--version"]); - self.cargo_tool_version(command).await - } - - pub async fn version_miri(&self) -> Result<Version> { - let mut command = basic_secure_docker_command(); - command.args(&["miri", "cargo", "miri", "--version"]); - self.cargo_tool_version(command).await - } - - // Parses versions of the shape `toolname 0.0.0 (0000000 0000-00-00)` - async fn cargo_tool_version(&self, command: Command) -> Result<Version> { - let output = run_command_with_timeout(command).await?; - let version_output = vec_to_str(output.stdout)?; - let mut parts = version_output.split_whitespace().fuse().skip(1); - - let release = parts.next().unwrap_or("").into(); - let commit_hash = parts.next().unwrap_or("").trim_start_matches('(').into(); - let commit_date = parts.next().unwrap_or("").trim_end_matches(')').into(); - - Ok(Version { - release, - commit_hash, - commit_date, - }) - } } async fn run_command_with_timeout(mut command: Command) -> Result<std::process::Output> { diff --git a/ui/src/server_axum.rs b/ui/src/server_axum.rs index f4101c95a..17acf3ee9 100644 --- a/ui/src/server_axum.rs +++ b/ui/src/server_axum.rs @@ -4,14 +4,14 @@ use crate::{ record_metric, track_metric_no_request_async, Endpoint, HasLabelsCore, Outcome, UNAVAILABLE_WS, }, - sandbox::{Channel, Sandbox, DOCKER_PROCESS_TIMEOUT_SOFT}, + sandbox::{Sandbox, DOCKER_PROCESS_TIMEOUT_SOFT}, CachingSnafu, ClippyRequest, ClippyResponse, ClippySnafu, CompileRequest, CompileResponse, CompileSnafu, Config, Error, ErrorJson, EvaluateRequest, EvaluateResponse, EvaluateSnafu, ExecuteRequest, ExecuteResponse, ExecuteSnafu, FormatRequest, FormatResponse, FormatSnafu, GhToken, GistCreationSnafu, GistLoadingSnafu, MacroExpansionRequest, MacroExpansionResponse, MacroExpansionSnafu, MetaCratesResponse, MetaGistCreateRequest, MetaGistResponse, - MetaVersionResponse, MetricsToken, MiriRequest, MiriResponse, MiriSnafu, Result, - SandboxCreationSnafu, ShutdownCoordinatorSnafu, TimeoutSnafu, + MetaVersionResponse, MetricsToken, MiriRequest, MiriResponse, MiriSnafu, MiriVersionSnafu, + Result, SandboxCreationSnafu, ShutdownCoordinatorSnafu, TimeoutSnafu, VersionsSnafu, }; use async_trait::async_trait; use axum::{ @@ -27,7 +27,7 @@ use axum::{ Router, }; use futures::{future::BoxFuture, FutureExt}; -use orchestrator::coordinator::{self, DockerBackend}; +use orchestrator::coordinator::{self, Coordinator, DockerBackend, Versions}; use snafu::prelude::*; use std::{ convert::TryInto, @@ -556,12 +556,7 @@ type Stamped<T> = (T, SystemTime); #[derive(Debug, Default)] struct SandboxCache { crates: CacheOne<MetaCratesResponse>, - version_stable: CacheOne<MetaVersionResponse>, - version_beta: CacheOne<MetaVersionResponse>, - version_nightly: CacheOne<MetaVersionResponse>, - version_rustfmt: CacheOne<MetaVersionResponse>, - version_clippy: CacheOne<MetaVersionResponse>, - version_miri: CacheOne<MetaVersionResponse>, + versions: CacheOne<Arc<Versions>>, } impl SandboxCache { @@ -573,65 +568,53 @@ impl SandboxCache { .await } - async fn version_stable(&self) -> Result<Stamped<MetaVersionResponse>> { - self.version_stable - .fetch(|sandbox| async move { - let version = sandbox - .version(Channel::Stable) - .await - .context(CachingSnafu)?; - Ok(version.into()) + async fn versions(&self) -> Result<Stamped<Arc<Versions>>> { + let coordinator = Coordinator::new_docker().await; + + self.versions + .fetch2(|| async { + Ok(Arc::new( + coordinator.versions().await.context(VersionsSnafu)?, + )) }) .await } + async fn version_stable(&self) -> Result<Stamped<MetaVersionResponse>> { + let (v, t) = self.versions().await?; + let v = (&v.stable.rustc).into(); + Ok((v, t)) + } + async fn version_beta(&self) -> Result<Stamped<MetaVersionResponse>> { - self.version_beta - .fetch(|sandbox| async move { - let version = sandbox.version(Channel::Beta).await.context(CachingSnafu)?; - Ok(version.into()) - }) - .await + let (v, t) = self.versions().await?; + let v = (&v.beta.rustc).into(); + Ok((v, t)) } async fn version_nightly(&self) -> Result<Stamped<MetaVersionResponse>> { - self.version_nightly - .fetch(|sandbox| async move { - let version = sandbox - .version(Channel::Nightly) - .await - .context(CachingSnafu)?; - Ok(version.into()) - }) - .await + let (v, t) = self.versions().await?; + let v = (&v.nightly.rustc).into(); + Ok((v, t)) } async fn version_rustfmt(&self) -> Result<Stamped<MetaVersionResponse>> { - self.version_rustfmt - .fetch(|sandbox| async move { - Ok(sandbox - .version_rustfmt() - .await - .context(CachingSnafu)? - .into()) - }) - .await + let (v, t) = self.versions().await?; + let v = (&v.nightly.rustfmt).into(); + Ok((v, t)) } async fn version_clippy(&self) -> Result<Stamped<MetaVersionResponse>> { - self.version_clippy - .fetch(|sandbox| async move { - Ok(sandbox.version_clippy().await.context(CachingSnafu)?.into()) - }) - .await + let (v, t) = self.versions().await?; + let v = (&v.nightly.clippy).into(); + Ok((v, t)) } async fn version_miri(&self) -> Result<Stamped<MetaVersionResponse>> { - self.version_miri - .fetch(|sandbox| async move { - Ok(sandbox.version_miri().await.context(CachingSnafu)?.into()) - }) - .await + let (v, t) = self.versions().await?; + let v = v.nightly.miri.as_ref().context(MiriVersionSnafu)?; + let v = v.into(); + Ok((v, t)) } } @@ -698,6 +681,59 @@ where Ok(value) } + + async fn fetch2<F, FFut>(&self, generator: F) -> Result<Stamped<T>> + where + F: FnOnce() -> FFut, + FFut: Future<Output = Result<T>>, + { + let data = &mut *self.0.lock().await; + match data { + Some(info) => { + if info.validation_time.elapsed() <= SANDBOX_CACHE_TIME_TO_LIVE { + Ok(info.stamped_value()) + } else { + Self::set_value2(data, generator).await + } + } + None => Self::set_value2(data, generator).await, + } + } + + async fn set_value2<F, FFut>( + data: &mut Option<CacheInfo<T>>, + generator: F, + ) -> Result<Stamped<T>> + where + F: FnOnce() -> FFut, + FFut: Future<Output = Result<T>>, + { + let value = generator().await?; + + let old_info = data.take(); + let new_info = CacheInfo::build(value); + + let info = match old_info { + Some(mut old_value) => { + if old_value.value == new_info.value { + // The value hasn't changed; record that we have + // checked recently, but keep the creation time to + // preserve caching. + old_value.validation_time = new_info.validation_time; + old_value + } else { + new_info + } + } + None => new_info, + }; + + let value = info.stamped_value(); + + *data = Some(info); + + Ok(value) + } } #[derive(Debug)] @@ -776,6 +812,16 @@ pub(crate) mod api_orchestrator_integration_impls { use snafu::prelude::*; use std::convert::TryFrom; + impl From<&Version> for crate::MetaVersionResponse { + fn from(other: &Version) -> Self { + Self { + version: (&*other.release).into(), + hash: (&*other.commit_hash).into(), + date: (&*other.commit_date).into(), + } + } + } + impl TryFrom<crate::EvaluateRequest> for ExecuteRequest { type Error = ParseEvaluateRequestError;