From d77e887f84eeaddf11efc8ec52e07abc79161405 Mon Sep 17 00:00:00 2001 From: Wolf Vollprecht Date: Sun, 18 Feb 2024 08:27:19 +0100 Subject: [PATCH] refactor: `script` execution during build (#641) --- mkdocs.yml | 2 +- src/build.rs | 226 +-------- src/env_vars.rs | 94 ---- src/lib.rs | 1 + src/package_test/run_test.rs | 272 +++++------ src/recipe/parser/script.rs | 8 +- src/script.rs | 441 ++++++++++++++++++ .../recipes/crazy_characters/recipe.yaml | 2 +- .../test-execution/recipe-test-succeed.yaml | 8 + test-data/recipes/test-execution/testfile.txt | 1 + .../test-execution/testfolder/data.txt | 1 + 11 files changed, 565 insertions(+), 491 deletions(-) create mode 100644 src/script.rs create mode 100644 test-data/recipes/test-execution/testfile.txt create mode 100644 test-data/recipes/test-execution/testfolder/data.txt diff --git a/mkdocs.yml b/mkdocs.yml index e9ac3c5b0..6415f985a 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -19,7 +19,7 @@ theme: accent: prefix toggle: - icon: material/brightness-7 + icon: material/brightness-7 name: Switch to dark mode # Palette toggle for dark mode diff --git a/src/build.rs b/src/build.rs index 7363c4b42..97ef8df93 100644 --- a/src/build.rs +++ b/src/build.rs @@ -1,205 +1,18 @@ //! The build module contains the code for running the build process for a given [`Output`] -use std::ffi::OsString; - -use std::io::{BufRead, BufReader, ErrorKind, Write}; - use fs_err as fs; -use fs_err::File; -use std::borrow::Cow; -use std::path::Path; use std::path::PathBuf; -use std::process::{Command, Stdio}; -use itertools::Itertools; use miette::IntoDiagnostic; use rattler_index::index; -use rattler_shell::shell; -use crate::env_vars::write_env_script; -use crate::metadata::{Directories, Output}; +use crate::metadata::Output; use crate::package_test::TestConfiguration; use crate::packaging::{package_conda, Files}; -use crate::recipe::parser::{ScriptContent, TestType}; +use crate::recipe::parser::TestType; use crate::render::resolved_dependencies::{install_environments, resolve_dependencies}; use crate::source::fetch_sources; use crate::{package_test, tool_configuration}; -const BASH_PREAMBLE: &str = r#" -## Start of bash preamble -if [ -z ${CONDA_BUILD+x} ]; then - source ((script_path)) -fi -# enable debug mode for the rest of the script -set -x -## End of preamble -"#; - -/// Create a conda build script and return the path to it -pub fn get_conda_build_script( - output: &Output, - directories: &Directories, -) -> Result { - let recipe = &output.recipe; - - let script = recipe.build().script(); - let default_extension = if output.build_configuration.target_platform.is_windows() { - "bat" - } else { - "sh" - }; - let script_content = match script.contents() { - // No script was specified, so we try to read the default script. If the file cannot be - // found we return an empty string. - ScriptContent::Default => { - let recipe_file = directories - .recipe_dir - .join(Path::new("build").with_extension(default_extension)); - match std::fs::read_to_string(recipe_file) { - Err(err) if err.kind() == ErrorKind::NotFound => String::new(), - Err(e) => { - return Err(e); - } - Ok(content) => content, - } - } - - // The scripts path was explicitly specified. If the file cannot be found we error out. - ScriptContent::Path(path) => { - let path_with_ext = if path.extension().is_none() { - Cow::Owned(path.with_extension(default_extension)) - } else { - Cow::Borrowed(path.as_path()) - }; - let recipe_file = directories.recipe_dir.join(path_with_ext); - match std::fs::read_to_string(&recipe_file) { - Err(err) if err.kind() == ErrorKind::NotFound => { - return Err(std::io::Error::new( - std::io::ErrorKind::NotFound, - format!("recipe file {:?} does not exist", recipe_file.display()), - )); - } - Err(e) => { - return Err(e); - } - Ok(content) => content, - } - } - // The scripts content was specified but it is still ambiguous whether it is a path or the - // contents of the string. Try to read the file as a script but fall back to using the string - // as the contents itself if the file is missing. - ScriptContent::CommandOrPath(path) => { - let content = - if !path.contains('\n') && (path.ends_with(".bat") || path.ends_with(".sh")) { - let recipe_file = directories.recipe_dir.join(Path::new(path)); - match std::fs::read_to_string(recipe_file) { - Err(err) if err.kind() == ErrorKind::NotFound => None, - Err(e) => { - return Err(e); - } - Ok(content) => Some(content), - } - } else { - None - }; - match content { - Some(content) => content, - None => path.to_owned(), - } - } - ScriptContent::Commands(commands) => commands.iter().join("\n"), - ScriptContent::Command(command) => command.to_owned(), - }; - - if script.interpreter().is_some() { - // We don't support an interpreter yet - tracing::error!("build.script.interpreter is not supported yet"); - } - - if cfg!(unix) { - let build_env_script_path = directories.work_dir.join("build_env.sh"); - let preamble = - BASH_PREAMBLE.replace("((script_path))", &build_env_script_path.to_string_lossy()); - - let mut file_out = File::create(&build_env_script_path)?; - write_env_script(output, "BUILD", &mut file_out, shell::Bash).map_err(|e| { - std::io::Error::new( - std::io::ErrorKind::Other, - format!("Failed to write build env script: {}", e), - ) - })?; - let full_script = format!("{}\n{}", preamble, script_content); - let build_script_path = directories.work_dir.join("conda_build.sh"); - - let mut build_script_file = File::create(&build_script_path)?; - build_script_file.write_all(full_script.as_bytes())?; - Ok(build_script_path) - } else { - let build_env_script_path = directories.work_dir.join("build_env.bat"); - let preamble = format!( - "IF \"%CONDA_BUILD%\" == \"\" (\n call {}\n)", - build_env_script_path.to_string_lossy() - ); - let mut file_out = File::create(&build_env_script_path)?; - - write_env_script(output, "BUILD", &mut file_out, shell::CmdExe).map_err(|e| { - std::io::Error::new( - std::io::ErrorKind::Other, - format!("Failed to write build env script: {}", e), - ) - })?; - - let full_script = format!("{}\n{}", preamble, script_content); - let build_script_path = directories.work_dir.join("conda_build.bat"); - - let mut build_script_file = File::create(&build_script_path)?; - build_script_file.write_all(full_script.as_bytes())?; - Ok(build_script_path) - } -} - -/// Spawns a process and replaces the given strings in the output with the given replacements. -/// This is used to replace the host prefix with $PREFIX and the build prefix with $BUILD_PREFIX -fn run_process_with_replacements( - command: &str, - cwd: &PathBuf, - args: &[OsString], - replacements: &[(&str, &str)], -) -> miette::Result<()> { - let (reader, writer) = os_pipe::pipe().expect("Could not get pipe"); - let writer_clone = writer.try_clone().expect("Could not clone writer pipe"); - - let mut child = Command::new(command) - .current_dir(cwd) - .args(args) - .stdin(Stdio::null()) - .stdout(writer) - .stderr(writer_clone) - .spawn() - .expect("Failed to execute command"); - - let reader = BufReader::new(reader); - - // Process the output line by line - for line in reader.lines() { - if let Ok(line) = line { - let filtered_line = replacements - .iter() - .fold(line, |acc, (from, to)| acc.replace(from, to)); - tracing::info!("{}", filtered_line); - } else { - tracing::warn!("Error reading output: {:?}", line); - } - } - - let status = child.wait().expect("Failed to wait on child"); - - if !status.success() { - return Err(miette::miette!("Build failed")); - } - - Ok(()) -} - /// Run the build for the given output. This will fetch the sources, resolve the dependencies, /// and execute the build script. Returns the path to the resulting package. pub async fn run_build( @@ -266,40 +79,7 @@ pub async fn run_build( } }; - let build_script = get_conda_build_script(&output, directories).into_diagnostic()?; - tracing::info!("Work dir: {:?}", &directories.work_dir); - tracing::info!("Build script: {:?}", build_script); - - let (interpreter, args) = if cfg!(unix) { - ( - "/bin/bash", - vec![OsString::from("-e"), build_script.as_os_str().to_owned()], - ) - } else { - ( - "cmd.exe", - vec![ - OsString::from("/d"), - OsString::from("/c"), - build_script.as_os_str().to_owned(), - ], - ) - }; - run_process_with_replacements( - interpreter, - &directories.work_dir, - &args, - &[ - ( - directories.host_prefix.to_string_lossy().as_ref(), - "$PREFIX", - ), - ( - directories.build_prefix.to_string_lossy().as_ref(), - "$BUILD_PREFIX", - ), - ], - )?; + output.run_build_script().await.into_diagnostic()?; let files_after = Files::from_prefix( &directories.host_prefix, diff --git a/src/env_vars.rs b/src/env_vars.rs index a645b4be4..ac529fba0 100644 --- a/src/env_vars.rs +++ b/src/env_vars.rs @@ -4,8 +4,6 @@ use std::path::{Path, PathBuf}; use std::{collections::HashMap, env}; use rattler_conda_types::Platform; -use rattler_shell::activation::{ActivationError, ActivationVariables, Activator}; -use rattler_shell::shell::Shell; use crate::linux; use crate::macos; @@ -340,95 +338,3 @@ pub fn vars(output: &Output, build_state: &str) -> HashMap { vars } - -#[derive(thiserror::Error, Debug)] -pub enum ScriptError { - #[error("Failed to write build env script")] - WriteBuildEnv(#[from] std::io::Error), - - #[error("Failed to write activate script")] - WriteActivation(#[from] std::fmt::Error), - - #[error("Failed to create activation script")] - CreateActivation(#[from] ActivationError), -} - -/// Write a script that can be sourced to set the environment variables for the build process. -/// The script will also activate the host and build prefixes. -pub fn write_env_script( - output: &Output, - state: &str, - out: &mut impl std::io::Write, - shell_type: T, -) -> Result<(), ScriptError> { - let directories = &output.build_configuration.directories; - - let vars = vars(output, state); - let mut s = String::new(); - for v in vars { - shell_type.set_env_var(&mut s, &v.0, &v.1)?; - } - - let platform = output.build_configuration.target_platform; - - let additional_os_vars = os_vars(&directories.host_prefix, &platform); - - for (k, v) in additional_os_vars { - shell_type.set_env_var(&mut s, &k, &v)?; - } - - for (k, v) in output.recipe.build().script().env() { - shell_type.set_env_var(&mut s, k, v)?; - } - - if !output.recipe.build().script().secrets().is_empty() { - tracing::error!("Secrets are not supported yet"); - } - - writeln!(out, "{}", s)?; - - let host_prefix_activator = Activator::from_path( - &directories.host_prefix, - shell_type.clone(), - output.build_configuration.build_platform, - )?; - let current_path = std::env::var("PATH") - .ok() - .map(|p| std::env::split_paths(&p).collect::>()); - - // if we are in a conda environment, we need to deactivate it before activating the host / build prefix - let conda_prefix = std::env::var("CONDA_PREFIX").ok().map(|p| p.into()); - - let activation_vars = ActivationVariables { - conda_prefix, - path: current_path, - path_modification_behavior: Default::default(), - }; - - let host_activation = host_prefix_activator - .activation(activation_vars) - .expect("Could not activate host prefix"); - - let build_prefix_activator = Activator::from_path( - &directories.build_prefix, - shell_type, - output.build_configuration.build_platform, - )?; - - // We use the previous PATH and _no_ CONDA_PREFIX to stack the build - // prefix on top of the host prefix - let activation_vars = ActivationVariables { - conda_prefix: None, - path: Some(host_activation.path.clone()), - path_modification_behavior: Default::default(), - }; - - let build_activation = build_prefix_activator - .activation(activation_vars) - .expect("Could not activate host prefix"); - - writeln!(out, "{}", host_activation.script)?; - writeln!(out, "{}", build_activation.script)?; - - Ok(()) -} diff --git a/src/lib.rs b/src/lib.rs index b5a459018..4fd812816 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -8,6 +8,7 @@ pub mod package_test; pub mod packaging; pub mod recipe; pub mod render; +pub mod script; pub mod selectors; pub mod source; pub mod system_tools; diff --git a/src/package_test/run_test.rs b/src/package_test/run_test.rs index e39292f96..a6ebacc99 100644 --- a/src/package_test/run_test.rs +++ b/src/package_test/run_test.rs @@ -11,7 +11,6 @@ use fs_err as fs; use rattler_conda_types::package::IndexJson; use std::fmt::Write as fmt_write; use std::{ - io::Write, path::{Path, PathBuf}, str::FromStr, }; @@ -20,13 +19,11 @@ use dunce::canonicalize; use rattler::package_cache::CacheKey; use rattler_conda_types::{package::ArchiveIdentifier, MatchSpec, Platform}; use rattler_index::index; -use rattler_shell::{ - activation::{ActivationError, ActivationVariables, Activator}, - shell::{Shell, ShellEnum, ShellScript}, -}; +use rattler_shell::activation::ActivationError; +use crate::env_vars; +use crate::recipe::parser::{Script, ScriptContent}; use crate::{ - env_vars, recipe::parser::{CommandsTestRequirements, PythonTest}, render::solver::create_environment, tool_configuration, @@ -50,8 +47,8 @@ pub enum TestError { #[error("failed to run test")] TestFailed, - #[error("failed to read package: {0}")] - PackageRead(#[from] std::io::Error), + #[error(transparent)] + IoError(#[from] std::io::Error), #[error("failed to write testing script: {0}")] FailedToWriteScript(#[from] std::fmt::Error), @@ -87,143 +84,57 @@ enum Tests { Python(PathBuf), } -fn run_in_environment( - shell: ShellEnum, - cmd: String, - cwd: &Path, - environment: &Path, - build_environment: Option, -) -> Result<(), TestError> { - let current_path = std::env::var("PATH") - .ok() - .map(|p| std::env::split_paths(&p).collect::>()); - - // if we are in a conda environment, we need to deactivate it before activating the host / build prefix - let conda_prefix = std::env::var("CONDA_PREFIX").ok().map(|p| p.into()); - - let activation_vars = ActivationVariables { - conda_prefix, - path: current_path, - path_modification_behavior: Default::default(), - }; - - let host_prefix_activator = - Activator::from_path(environment, shell.clone(), Platform::current())?; - - let host_activation = host_prefix_activator - .activation(activation_vars) - .expect("Could not activate host prefix"); - - let build_activation_script = if let Some(build_environment) = build_environment { - // We use the previous PATH and _no_ CONDA_PREFIX to stack the build - // prefix on top of the host prefix - let activation_vars = ActivationVariables { - conda_prefix: None, - path: Some(host_activation.path.clone()), - path_modification_behavior: Default::default(), - }; - - let activator = - Activator::from_path(&build_environment, shell.clone(), Platform::current())?; - activator.activation(activation_vars)?.script - } else { - String::new() - }; - - let mut script_content = String::new(); - let mut additional_script = ShellScript::new(shell.clone(), Platform::current()); - - let os_vars = env_vars::os_vars(environment, &Platform::current()); - for (key, val) in os_vars { - if key == "PATH" { - continue; - } - additional_script.set_env_var(&key, &val); - } - - additional_script.set_env_var("PREFIX", environment.to_string_lossy().as_ref()); - writeln!(script_content, "{}", additional_script.contents)?; - writeln!(script_content, "{}", host_activation.script)?; - writeln!(script_content, "{}", build_activation_script)?; - if matches!(shell, ShellEnum::Bash(_)) { - writeln!(script_content, "set -x")?; - } - writeln!(script_content, "{}", cmd)?; - - let mut tmpfile = tempfile::Builder::new() - .prefix("rattler-test-") - .suffix(&format!(".{}", shell.extension())) - .tempfile()?; - - if matches!(shell, ShellEnum::CmdExe(_)) { - script_content = format!("chcp 65001 > nul\n{script_content}").replace('\n', "\r\n"); - tmpfile.write_all(script_content.as_bytes())?; - } else { - tmpfile.write_all(script_content.as_bytes())?; - } - - let tmpfile_path = tmpfile.into_temp_path(); - - tracing::info!("Running test script:\n{}", script_content); - - let executable = shell.executable(); - let status = match shell { - ShellEnum::Bash(_) => std::process::Command::new(executable) - .arg("-e") - .arg(&tmpfile_path) - .current_dir(cwd) - .status()?, - ShellEnum::CmdExe(_) => std::process::Command::new(executable) - .arg("/d") - .arg("/c") - .arg(&tmpfile_path) - .current_dir(cwd) - .status()?, - _ => todo!("No shells implemented beyond cmd.exe and bash"), - }; - - if !status.success() { - return Err(TestError::TestFailed); - } - - Ok(()) -} - impl Tests { - fn run(&self, environment: &Path, cwd: &Path) -> Result<(), TestError> { - let default_shell = ShellEnum::default(); + async fn run(&self, environment: &Path, cwd: &Path) -> Result<(), TestError> { + tracing::info!("Testing commands:"); + + let mut env_vars = env_vars::os_vars(environment, &Platform::current()); + env_vars.retain(|key, _| key != "PATH"); + env_vars.insert( + "PREFIX".to_string(), + environment.to_string_lossy().to_string(), + ); + let tmp_dir = tempfile::tempdir()?; match self { Tests::Commands(path) => { - let contents = fs::read_to_string(path)?; - let is_path_ext = - |ext: &str| path.extension().map(|s| s.eq(ext)).unwrap_or_default(); - if Platform::current().is_windows() && is_path_ext("bat") { - tracing::info!("Testing commands:"); - run_in_environment(default_shell, contents, cwd, environment, None) - } else if Platform::current().is_unix() && is_path_ext("sh") { - tracing::info!("Testing commands:"); - run_in_environment(default_shell, contents, cwd, environment, None) - } else { - Ok(()) - } + let script = Script { + content: ScriptContent::Path(path.clone()), + ..Script::default() + }; + + // copy all test files to a temporary directory and set it as the working directory + let copy_options = fs_extra::dir::CopyOptions::new().content_only(true); + fs_extra::dir::copy(path, tmp_dir.path(), ©_options).map_err(|e| { + TestError::IoError(std::io::Error::new( + std::io::ErrorKind::Other, + format!("Failed to copy test files: {}", e), + )) + })?; + + script + .run_script(env_vars, tmp_dir.path(), cwd, environment, None) + .await + .map_err(|_| TestError::TestFailed)?; } Tests::Python(path) => { - let imports = fs::read_to_string(path)?; - tracing::info!("Testing Python imports:\n{imports}"); - run_in_environment( - default_shell, - format!("python {}", path.to_string_lossy()), - cwd, - environment, - None, - ) + let script = Script { + content: ScriptContent::Path(path.clone()), + interpreter: Some("python".into()), + ..Script::default() + }; + + script + .run_script(env_vars, tmp_dir.path(), cwd, environment, None) + .await + .map_err(|_| TestError::TestFailed)?; } } + Ok(()) } } -async fn legacy_tests_from_folder(pkg: &Path) -> Result<(PathBuf, Vec), TestError> { +async fn legacy_tests_from_folder(pkg: &Path) -> Result<(PathBuf, Vec), std::io::Error> { let mut tests = Vec::new(); let test_folder = pkg.join("info/test"); @@ -243,10 +154,10 @@ async fn legacy_tests_from_folder(pkg: &Path) -> Result<(PathBuf, Vec), T continue; }; if file_name.eq("run_test.sh") || file_name.eq("run_test.bat") { - println!("test {}", file_name.to_string_lossy()); + tracing::info!("test {}", file_name.to_string_lossy()); tests.push(Tests::Commands(path)); } else if file_name.eq("run_test.py") { - println!("test {}", file_name.to_string_lossy()); + tracing::info!("test {}", file_name.to_string_lossy()); tests.push(Tests::Python(path)); } } @@ -398,7 +309,7 @@ pub async fn run_test(package_file: &Path, config: &TestConfiguration) -> Result let (test_folder, tests) = legacy_tests_from_folder(&package_folder).await?; for test in tests { - test.run(&prefix, &test_folder)?; + test.run(&prefix, &test_folder).await?; } tracing::info!( @@ -414,7 +325,7 @@ pub async fn run_test(package_file: &Path, config: &TestConfiguration) -> Result // for each enumerated test, we load and run it while let Some(entry) = read_dir.next_entry().await? { - println!("test {:?}", entry.path()); + tracing::info!("test {:?}", entry.path()); run_individual_test(&pkg, &entry.path(), &prefix, &config).await?; } @@ -436,7 +347,7 @@ async fn run_python_test( config: &TestConfiguration, ) -> Result<(), TestError> { let test_file = path.join("python_test.json"); - let test: PythonTest = serde_json::from_str(&fs::read_to_string(test_file)?)?; + let test: PythonTest = serde_json::from_reader(fs::File::open(test_file)?)?; let match_spec = MatchSpec::from_str(format!("{}={}={}", pkg.name, pkg.version, pkg.build_string).as_str()) @@ -446,11 +357,9 @@ async fn run_python_test( dependencies.push(MatchSpec::from_str("pip").unwrap()); } - let platform = Platform::current(); - create_environment( &dependencies, - &platform, + &Platform::current(), prefix, &config.channels, &config.tool_configuration, @@ -458,30 +367,45 @@ async fn run_python_test( .await .map_err(TestError::TestEnvironmentSetup)?; - let default_shell = ShellEnum::default(); - - let mut test_file = tempfile::Builder::new() - .prefix("rattler-test-") - .suffix(".py") - .tempfile()?; - + let mut imports = String::new(); for import in test.imports { - writeln!(test_file, "import {}", import)?; + writeln!(imports, "import {}", import)?; } - run_in_environment( - default_shell.clone(), - format!("python {}", test_file.path().to_string_lossy()), - path, - prefix, - None, - )?; + let script = Script { + content: ScriptContent::Command(imports), + interpreter: Some("python".into()), + ..Script::default() + }; + + let tmp_dir = tempfile::tempdir()?; + script + .run_script(Default::default(), tmp_dir.path(), path, prefix, None) + .await + .map_err(|_| TestError::TestFailed)?; + + tracing::info!( + "{} python imports test passed!", + console::style(console::Emoji("✔", "")).green() + ); if test.pip_check { - run_in_environment(default_shell, "pip check".into(), path, prefix, None) - } else { - Ok(()) + let script = Script { + content: ScriptContent::Command("pip check".into()), + ..Script::default() + }; + script + .run_script(Default::default(), path, path, prefix, None) + .await + .map_err(|_| TestError::TestFailed)?; + + tracing::info!( + "{} pip check passed!", + console::style(console::Emoji("✔", "")).green() + ); } + + Ok(()) } async fn run_shell_test( @@ -545,18 +469,30 @@ async fn run_shell_test( .await .map_err(TestError::TestEnvironmentSetup)?; - let default_shell = ShellEnum::default(); + let mut env_vars = env_vars::os_vars(prefix, &Platform::current()); + env_vars.retain(|key, _| key != "PATH"); + env_vars.insert("PREFIX".to_string(), run_env.to_string_lossy().to_string()); - let test_file_path = if platform.is_windows() { - path.join("run_test.bat") - } else { - path.join("run_test.sh") + let script = Script { + content: ScriptContent::Path(PathBuf::from("run_test")), + ..Default::default() }; - let contents = fs::read_to_string(test_file_path)?; + // copy all test files to a temporary directory and set it as the working directory + let tmp_dir = tempfile::tempdir()?; + let copy_options = fs_extra::dir::CopyOptions::new().content_only(true); + fs_extra::dir::copy(path, tmp_dir.path(), ©_options).map_err(|e| { + TestError::IoError(std::io::Error::new( + std::io::ErrorKind::Other, + format!("Failed to copy test files: {}", e), + )) + })?; tracing::info!("Testing commands:"); - run_in_environment(default_shell, contents, path, &run_env, build_env)?; + script + .run_script(env_vars, tmp_dir.path(), path, &run_env, build_env.as_ref()) + .await + .map_err(|_| TestError::TestFailed)?; Ok(()) } @@ -576,7 +512,7 @@ async fn run_individual_test( // no test found } - println!( + tracing::info!( "{} test passed!", console::style(console::Emoji("✔", "")).green() ); diff --git a/src/recipe/parser/script.rs b/src/recipe/parser/script.rs index 8df860711..7f499668a 100644 --- a/src/recipe/parser/script.rs +++ b/src/recipe/parser/script.rs @@ -13,15 +13,15 @@ use std::{borrow::Cow, collections::BTreeMap, path::PathBuf}; #[derive(Debug, Default, Clone)] pub struct Script { /// The interpreter to use for the script. - pub(super) interpreter: Option, + pub interpreter: Option, /// Environment variables to set in the build environment. - pub(super) env: BTreeMap, + pub env: BTreeMap, /// Environment variables to leak into the build environment from the host system that /// contain sensitve information. Use with care because this might make recipes no /// longer reproducible on other machines. - pub(super) secrets: Vec, + pub secrets: Vec, /// The contents of the script, either a path or a list of commands. - pub(super) content: ScriptContent, + pub content: ScriptContent, } impl Serialize for Script { diff --git a/src/script.rs b/src/script.rs new file mode 100644 index 000000000..0539b220d --- /dev/null +++ b/src/script.rs @@ -0,0 +1,441 @@ +#![allow(missing_docs)] +use indexmap::IndexMap; +use itertools::Itertools; +use rattler_conda_types::Platform; +use rattler_shell::{ + activation::{ActivationError, ActivationVariables, Activator}, + shell::{self, Shell}, +}; +use std::{ + borrow::Cow, + collections::HashMap, + fmt::Write as WriteFmt, + io::ErrorKind, + path::{Path, PathBuf}, + process::Stdio, +}; +use tokio::io::AsyncBufReadExt as _; + +use crate::{ + env_vars::{self}, + metadata::Output, + recipe::parser::{Script, ScriptContent}, +}; + +const BASH_PREAMBLE: &str = r#" +## Start of bash preamble +if [ -z ${CONDA_BUILD+x} ]; then + source ((script_path)) +fi +# enable debug mode for the rest of the script +set -x +## End of preamble +"#; + +pub struct ExecutionArgs { + pub script: String, + pub env_vars: IndexMap, + pub secrets: IndexMap, + + pub execution_platform: Platform, + + pub build_prefix: Option, + pub run_prefix: PathBuf, + + pub work_dir: PathBuf, +} + +impl ExecutionArgs { + /// Returns strings that should be replaced. The template argument can be used to specify + /// a nice "variable" syntax, e.g. "$((var))" for bash or "%((var))%" for cmd.exe. The `var` part + /// will be replaced with the actual variable name. + pub fn replacements(&self, template: &str) -> HashMap { + let mut replacements = HashMap::new(); + if let Some(build_prefix) = &self.build_prefix { + replacements.insert( + build_prefix.to_string_lossy().to_string(), + template.replace("((var))", "BUILD_PREFIX"), + ); + }; + replacements.insert( + self.run_prefix.to_string_lossy().to_string(), + template.replace("((var))", "PREFIX"), + ); + + self.secrets.iter().for_each(|(_, v)| { + replacements.insert(v.to_string(), "********".to_string()); + }); + + replacements + } +} + +trait Interpreter { + fn get_script( + &self, + args: &ExecutionArgs, + shell_type: T, + ) -> Result { + let mut shell_script = shell::ShellScript::new(shell_type, Platform::current()); + for (k, v) in args.env_vars.iter() { + shell_script.set_env_var(k, v); + } + let host_prefix_activator = + Activator::from_path(&args.run_prefix, shell_type, args.execution_platform)?; + + let current_path = std::env::var("PATH") + .ok() + .map(|p| std::env::split_paths(&p).collect::>()); + let conda_prefix = std::env::var("CONDA_PREFIX").ok().map(|p| p.into()); + + let activation_vars = ActivationVariables { + conda_prefix, + path: current_path, + path_modification_behavior: Default::default(), + }; + + let host_activation = host_prefix_activator.activation(activation_vars)?; + + if let Some(build_prefix) = &args.build_prefix { + let build_prefix_activator = + Activator::from_path(build_prefix, shell_type, args.execution_platform)?; + + let activation_vars = ActivationVariables { + conda_prefix: None, + path: Some(host_activation.path.clone()), + path_modification_behavior: Default::default(), + }; + + let build_activation = build_prefix_activator.activation(activation_vars)?; + + writeln!(shell_script.contents, "{}", host_activation.script)?; + writeln!(shell_script.contents, "{}", build_activation.script)?; + } else { + writeln!(shell_script.contents, "{}", host_activation.script)?; + } + + Ok(shell_script.contents) + } + + async fn run(&self, args: ExecutionArgs) -> Result<(), std::io::Error>; +} + +struct BashInterpreter; + +impl Interpreter for BashInterpreter { + async fn run(&self, args: ExecutionArgs) -> Result<(), std::io::Error> { + let script = self.get_script(&args, shell::Bash).unwrap(); + + let build_env_path = args.work_dir.join("build_env.sh"); + let build_script_path = args.work_dir.join("conda_build.sh"); + + tokio::fs::write(&build_env_path, script).await?; + + let preamble = BASH_PREAMBLE.replace("((script_path))", &build_env_path.to_string_lossy()); + let script = format!("{}\n{}", preamble, args.script); + tokio::fs::write(&build_script_path, script).await?; + + let build_script_path_str = build_script_path.to_string_lossy().to_string(); + let cmd_args = ["bash", "-e", &build_script_path_str]; + + let output = run_process_with_replacements( + &cmd_args, + &args.work_dir, + &args.replacements("$((var))"), + ) + .await?; + + if !output.status.success() { + return Err(std::io::Error::new( + std::io::ErrorKind::Other, + format!("Script failed with status {:?}", output.status), + )); + } + + Ok(()) + } +} + +const CMDEXE_PREAMBLE: &str = r#" +chcp 65001 > nul +IF "%CONDA_BUILD%" == "" ( + call ((script_path)) +) +"#; + +struct CmdExeInterpreter; + +impl Interpreter for CmdExeInterpreter { + async fn run(&self, args: ExecutionArgs) -> Result<(), std::io::Error> { + let script = self.get_script(&args, shell::CmdExe).unwrap(); + + let build_env_path = args.work_dir.join("build_env.bat"); + let build_script_path = args.work_dir.join("conda_build.bat"); + + tokio::fs::write(&build_env_path, &script).await?; + + let build_script = format!( + "{}\n{}", + CMDEXE_PREAMBLE.replace("((script_path))", &build_env_path.to_string_lossy()), + args.script + ); + tokio::fs::write(&build_script_path, &build_script).await?; + + let build_script_path_str = build_script_path.to_string_lossy().to_string(); + let cmd_args = ["cmd.exe", "/d", "/c", &build_script_path_str]; + + let output = run_process_with_replacements( + &cmd_args, + &args.work_dir, + &args.replacements("%((var))%"), + ) + .await?; + + if !output.status.success() { + return Err(std::io::Error::new( + std::io::ErrorKind::Other, + format!("Script failed with status {:?}", output.status), + )); + } + + Ok(()) + } +} + +struct PythonInterpreter; + +// python interpreter calls either bash or cmd.exe interpreter for activation and then runs python script +impl Interpreter for PythonInterpreter { + async fn run(&self, args: ExecutionArgs) -> Result<(), std::io::Error> { + let py_script = args.work_dir.join("conda_build_script.py"); + tokio::fs::write(&py_script, args.script).await?; + + let args = ExecutionArgs { + script: format!("python {:?}", py_script), + ..args + }; + + if cfg!(windows) { + CmdExeInterpreter.run(args).await + } else { + BashInterpreter.run(args).await + } + } +} + +impl Script { + fn get_contents(&self, recipe_dir: &Path) -> Result { + let default_extension = if cfg!(windows) { "bat" } else { "sh" }; + + let script_content = match self.contents() { + // No script was specified, so we try to read the default script. If the file cannot be + // found we return an empty string. + ScriptContent::Default => { + let recipe_file = + recipe_dir.join(Path::new("build").with_extension(default_extension)); + match std::fs::read_to_string(recipe_file) { + Err(err) if err.kind() == ErrorKind::NotFound => String::new(), + Err(e) => { + return Err(e); + } + Ok(content) => content, + } + } + + // The scripts path was explicitly specified. If the file cannot be found we error out. + ScriptContent::Path(path) => { + let path_with_ext = if path.extension().is_none() { + Cow::Owned(path.with_extension(default_extension)) + } else { + Cow::Borrowed(path.as_path()) + }; + let recipe_file = recipe_dir.join(path_with_ext); + match std::fs::read_to_string(&recipe_file) { + Err(err) if err.kind() == ErrorKind::NotFound => { + return Err(std::io::Error::new( + std::io::ErrorKind::NotFound, + format!("recipe file {:?} does not exist", recipe_file.display()), + )); + } + Err(e) => { + return Err(e); + } + Ok(content) => content, + } + } + // The scripts content was specified but it is still ambiguous whether it is a path or the + // contents of the string. Try to read the file as a script but fall back to using the string + // as the contents itself if the file is missing. + ScriptContent::CommandOrPath(path) => { + let content = + if !path.contains('\n') && (path.ends_with(".bat") || path.ends_with(".sh")) { + let recipe_file = recipe_dir.join(Path::new(path)); + match std::fs::read_to_string(recipe_file) { + Err(err) if err.kind() == ErrorKind::NotFound => None, + Err(e) => { + return Err(e); + } + Ok(content) => Some(content), + } + } else { + None + }; + match content { + Some(content) => content, + None => path.to_owned(), + } + } + ScriptContent::Commands(commands) => commands.iter().join("\n"), + ScriptContent::Command(command) => command.to_owned(), + }; + + Ok(script_content) + } + + pub async fn run_script( + &self, + env_vars: HashMap, + work_dir: &Path, + recipe_dir: &Path, + run_prefix: &Path, + build_prefix: Option<&PathBuf>, + ) -> Result<(), std::io::Error> { + let interpreter = self + .interpreter() + .unwrap_or(if cfg!(windows) { "cmd" } else { "bash" }); + + let contents = self.get_contents(recipe_dir)?; + + let secrets = self + .secrets() + .iter() + .filter_map(|k| { + let secret = k.to_string(); + + if let Ok(value) = std::env::var(&secret) { + Some((secret, value)) + } else { + tracing::warn!("Secret {} not found in environment", secret); + None + } + }) + .collect::>(); + + let env_vars = env_vars + .into_iter() + .chain(self.env().clone().into_iter()) + .collect::>(); + + let exec_args = ExecutionArgs { + script: contents, + env_vars, + secrets, + build_prefix: build_prefix.map(|p| p.to_owned()), + run_prefix: run_prefix.to_owned(), + execution_platform: Platform::current(), + work_dir: work_dir.to_owned(), + }; + + match interpreter { + "bash" => BashInterpreter.run(exec_args).await?, + "cmd" => CmdExeInterpreter.run(exec_args).await?, + "python" => PythonInterpreter.run(exec_args).await?, + _ => { + return Err(std::io::Error::new( + std::io::ErrorKind::Other, + format!("Unsupported interpreter: {}", interpreter), + )) + } + }; + + Ok(()) + } +} + +impl Output { + pub async fn run_build_script(&self) -> Result<(), std::io::Error> { + let host_prefix = self.build_configuration.directories.host_prefix.clone(); + let target_platform = self.build_configuration.target_platform; + let mut env_vars = env_vars::vars(self, "BUILD"); + env_vars.extend(env_vars::os_vars(&host_prefix, &target_platform)); + + self.recipe + .build() + .script() + .run_script( + env_vars, + &self.build_configuration.directories.work_dir, + &self.build_configuration.directories.recipe_dir, + &self.build_configuration.directories.host_prefix, + Some(&self.build_configuration.directories.build_prefix), + ) + .await?; + + Ok(()) + } +} + +/// Spawns a process and replaces the given strings in the output with the given replacements. +/// This is used to replace the host prefix with $PREFIX and the build prefix with $BUILD_PREFIX +async fn run_process_with_replacements( + args: &[&str], + cwd: &Path, + replacements: &HashMap, +) -> Result { + let mut command = tokio::process::Command::new(args[0]); + command + .current_dir(cwd) + .args(&args[1..]) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + let mut child = command.spawn()?; + + let stdout = child.stdout.take().expect("Failed to take stdout"); + let stderr = child.stderr.take().expect("Failed to take stderr"); + + let mut stdout_lines = tokio::io::BufReader::new(stdout).lines(); + let mut stderr_lines = tokio::io::BufReader::new(stderr).lines(); + + let mut stdout_log = String::new(); + let mut stderr_log = String::new(); + + loop { + let (line, is_stderr) = tokio::select! { + line = stdout_lines.next_line() => (line, false), + line = stderr_lines.next_line() => (line, true), + else => break, + }; + + match line { + Ok(Some(line)) => { + let filtered_line = replacements + .iter() + .fold(line, |acc, (from, to)| acc.replace(from, to)); + + if is_stderr { + stderr_log.push_str(&filtered_line); + stderr_log.push('\n'); + } else { + stdout_log.push_str(&filtered_line); + stdout_log.push('\n'); + } + + tracing::info!("{}", filtered_line); + } + Ok(None) => break, + Err(e) => { + tracing::warn!("Error reading output: {:?}", e); + } + }; + } + + let status = child.wait().await?; + + Ok(std::process::Output { + status, + stdout: stdout_log.into_bytes(), + stderr: stderr_log.into_bytes(), + }) +} diff --git a/test-data/recipes/crazy_characters/recipe.yaml b/test-data/recipes/crazy_characters/recipe.yaml index a32778c2f..cc96f03c4 100644 --- a/test-data/recipes/crazy_characters/recipe.yaml +++ b/test-data/recipes/crazy_characters/recipe.yaml @@ -31,4 +31,4 @@ tests: then: - if not exist "%PREFIX%\files\File(Glob …).tmSnippet" exit 1 - if not exist "%PREFIX%\files\a $random_crazy file name with spaces and (parentheses).txt" exit 1 - - if not exist %PREFIX%\files\a_really_long_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.txt exit 1 + - if not exist "%PREFIX%\files\a_really_long_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.txt" exit 1 diff --git a/test-data/recipes/test-execution/recipe-test-succeed.yaml b/test-data/recipes/test-execution/recipe-test-succeed.yaml index 99b0523a6..07fa291db 100644 --- a/test-data/recipes/test-execution/recipe-test-succeed.yaml +++ b/test-data/recipes/test-execution/recipe-test-succeed.yaml @@ -15,5 +15,13 @@ tests: - if: unix then: - test -f $PREFIX/test-execution.txt + - test -f ./testfile.txt + - test -f ./testfolder/data.txt else: - if not exist %PREFIX%\test-execution.txt (exit 1) + - if not exist .\testfile.txt (exit 1) + - if not exist .\testfolder\data.txt (exit 1) + files: + recipe: + - testfile.txt + - testfolder/ diff --git a/test-data/recipes/test-execution/testfile.txt b/test-data/recipes/test-execution/testfile.txt new file mode 100644 index 000000000..d173390cb --- /dev/null +++ b/test-data/recipes/test-execution/testfile.txt @@ -0,0 +1 @@ +just a test diff --git a/test-data/recipes/test-execution/testfolder/data.txt b/test-data/recipes/test-execution/testfolder/data.txt new file mode 100644 index 000000000..1269488f7 --- /dev/null +++ b/test-data/recipes/test-execution/testfolder/data.txt @@ -0,0 +1 @@ +data