Skip to content

Commit

Permalink
refactor: script execution during build (#641)
Browse files Browse the repository at this point in the history
  • Loading branch information
@wolfv
wolfv authored Feb 18, 2024
1 parent b49fffc commit d77e887
Show file tree
Hide file tree
Showing 11 changed files with 565 additions and 491 deletions.
2 changes: 1 addition & 1 deletion mkdocs.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ theme:
accent: prefix

toggle:
icon: material/brightness-7
icon: material/brightness-7
name: Switch to dark mode

# Palette toggle for dark mode
Expand Down
226 changes: 3 additions & 223 deletions src/build.rs
View file
Original file line number Diff line number Diff line change
@@ -1,205 +1,18 @@
//! The build module contains the code for running the build process for a given [`Output`]
use std::ffi::OsString;

use std::io::{BufRead, BufReader, ErrorKind, Write};

use fs_err as fs;
use fs_err::File;
use std::borrow::Cow;
use std::path::Path;
use std::path::PathBuf;
use std::process::{Command, Stdio};

use itertools::Itertools;
use miette::IntoDiagnostic;
use rattler_index::index;
use rattler_shell::shell;

use crate::env_vars::write_env_script;
use crate::metadata::{Directories, Output};
use crate::metadata::Output;
use crate::package_test::TestConfiguration;
use crate::packaging::{package_conda, Files};
use crate::recipe::parser::{ScriptContent, TestType};
use crate::recipe::parser::TestType;
use crate::render::resolved_dependencies::{install_environments, resolve_dependencies};
use crate::source::fetch_sources;
use crate::{package_test, tool_configuration};

const BASH_PREAMBLE: &str = r#"
## Start of bash preamble
if [ -z ${CONDA_BUILD+x} ]; then
source ((script_path))
fi
# enable debug mode for the rest of the script
set -x
## End of preamble
"#;

/// Create a conda build script and return the path to it
pub fn get_conda_build_script(
output: &Output,
directories: &Directories,
) -> Result<PathBuf, std::io::Error> {
let recipe = &output.recipe;

let script = recipe.build().script();
let default_extension = if output.build_configuration.target_platform.is_windows() {
"bat"
} else {
"sh"
};
let script_content = match script.contents() {
// No script was specified, so we try to read the default script. If the file cannot be
// found we return an empty string.
ScriptContent::Default => {
let recipe_file = directories
.recipe_dir
.join(Path::new("build").with_extension(default_extension));
match std::fs::read_to_string(recipe_file) {
Err(err) if err.kind() == ErrorKind::NotFound => String::new(),
Err(e) => {
return Err(e);
}
Ok(content) => content,
}
}

// The scripts path was explicitly specified. If the file cannot be found we error out.
ScriptContent::Path(path) => {
let path_with_ext = if path.extension().is_none() {
Cow::Owned(path.with_extension(default_extension))
} else {
Cow::Borrowed(path.as_path())
};
let recipe_file = directories.recipe_dir.join(path_with_ext);
match std::fs::read_to_string(&recipe_file) {
Err(err) if err.kind() == ErrorKind::NotFound => {
return Err(std::io::Error::new(
std::io::ErrorKind::NotFound,
format!("recipe file {:?} does not exist", recipe_file.display()),
));
}
Err(e) => {
return Err(e);
}
Ok(content) => content,
}
}
// The scripts content was specified but it is still ambiguous whether it is a path or the
// contents of the string. Try to read the file as a script but fall back to using the string
// as the contents itself if the file is missing.
ScriptContent::CommandOrPath(path) => {
let content =
if !path.contains('\n') && (path.ends_with(".bat") || path.ends_with(".sh")) {
let recipe_file = directories.recipe_dir.join(Path::new(path));
match std::fs::read_to_string(recipe_file) {
Err(err) if err.kind() == ErrorKind::NotFound => None,
Err(e) => {
return Err(e);
}
Ok(content) => Some(content),
}
} else {
None
};
match content {
Some(content) => content,
None => path.to_owned(),
}
}
ScriptContent::Commands(commands) => commands.iter().join("\n"),
ScriptContent::Command(command) => command.to_owned(),
};

if script.interpreter().is_some() {
// We don't support an interpreter yet
tracing::error!("build.script.interpreter is not supported yet");
}

if cfg!(unix) {
let build_env_script_path = directories.work_dir.join("build_env.sh");
let preamble =
BASH_PREAMBLE.replace("((script_path))", &build_env_script_path.to_string_lossy());

let mut file_out = File::create(&build_env_script_path)?;
write_env_script(output, "BUILD", &mut file_out, shell::Bash).map_err(|e| {
std::io::Error::new(
std::io::ErrorKind::Other,
format!("Failed to write build env script: {}", e),
)
})?;
let full_script = format!("{}\n{}", preamble, script_content);
let build_script_path = directories.work_dir.join("conda_build.sh");

let mut build_script_file = File::create(&build_script_path)?;
build_script_file.write_all(full_script.as_bytes())?;
Ok(build_script_path)
} else {
let build_env_script_path = directories.work_dir.join("build_env.bat");
let preamble = format!(
"IF \"%CONDA_BUILD%\" == \"\" (\n call {}\n)",
build_env_script_path.to_string_lossy()
);
let mut file_out = File::create(&build_env_script_path)?;

write_env_script(output, "BUILD", &mut file_out, shell::CmdExe).map_err(|e| {
std::io::Error::new(
std::io::ErrorKind::Other,
format!("Failed to write build env script: {}", e),
)
})?;

let full_script = format!("{}\n{}", preamble, script_content);
let build_script_path = directories.work_dir.join("conda_build.bat");

let mut build_script_file = File::create(&build_script_path)?;
build_script_file.write_all(full_script.as_bytes())?;
Ok(build_script_path)
}
}

/// Spawns a process and replaces the given strings in the output with the given replacements.
/// This is used to replace the host prefix with $PREFIX and the build prefix with $BUILD_PREFIX
fn run_process_with_replacements(
command: &str,
cwd: &PathBuf,
args: &[OsString],
replacements: &[(&str, &str)],
) -> miette::Result<()> {
let (reader, writer) = os_pipe::pipe().expect("Could not get pipe");
let writer_clone = writer.try_clone().expect("Could not clone writer pipe");

let mut child = Command::new(command)
.current_dir(cwd)
.args(args)
.stdin(Stdio::null())
.stdout(writer)
.stderr(writer_clone)
.spawn()
.expect("Failed to execute command");

let reader = BufReader::new(reader);

// Process the output line by line
for line in reader.lines() {
if let Ok(line) = line {
let filtered_line = replacements
.iter()
.fold(line, |acc, (from, to)| acc.replace(from, to));
tracing::info!("{}", filtered_line);
} else {
tracing::warn!("Error reading output: {:?}", line);
}
}

let status = child.wait().expect("Failed to wait on child");

if !status.success() {
return Err(miette::miette!("Build failed"));
}

Ok(())
}

/// Run the build for the given output. This will fetch the sources, resolve the dependencies,
/// and execute the build script. Returns the path to the resulting package.
pub async fn run_build(
Expand Down Expand Up @@ -266,40 +79,7 @@ pub async fn run_build(
}
};

let build_script = get_conda_build_script(&output, directories).into_diagnostic()?;
tracing::info!("Work dir: {:?}", &directories.work_dir);
tracing::info!("Build script: {:?}", build_script);

let (interpreter, args) = if cfg!(unix) {
(
"/bin/bash",
vec![OsString::from("-e"), build_script.as_os_str().to_owned()],
)
} else {
(
"cmd.exe",
vec![
OsString::from("/d"),
OsString::from("/c"),
build_script.as_os_str().to_owned(),
],
)
};
run_process_with_replacements(
interpreter,
&directories.work_dir,
&args,
&[
(
directories.host_prefix.to_string_lossy().as_ref(),
"$PREFIX",
),
(
directories.build_prefix.to_string_lossy().as_ref(),
"$BUILD_PREFIX",
),
],
)?;
output.run_build_script().await.into_diagnostic()?;

let files_after = Files::from_prefix(
&directories.host_prefix,
Expand Down
94 changes: 0 additions & 94 deletions src/env_vars.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,6 @@ use std::path::{Path, PathBuf};
use std::{collections::HashMap, env};

use rattler_conda_types::Platform;
use rattler_shell::activation::{ActivationError, ActivationVariables, Activator};
use rattler_shell::shell::Shell;

use crate::linux;
use crate::macos;
Expand Down Expand Up @@ -340,95 +338,3 @@ pub fn vars(output: &Output, build_state: &str) -> HashMap<String, String> {

vars
}

#[derive(thiserror::Error, Debug)]
pub enum ScriptError {
#[error("Failed to write build env script")]
WriteBuildEnv(#[from] std::io::Error),

#[error("Failed to write activate script")]
WriteActivation(#[from] std::fmt::Error),

#[error("Failed to create activation script")]
CreateActivation(#[from] ActivationError),
}

/// Write a script that can be sourced to set the environment variables for the build process.
/// The script will also activate the host and build prefixes.
pub fn write_env_script<T: Shell + Clone>(
output: &Output,
state: &str,
out: &mut impl std::io::Write,
shell_type: T,
) -> Result<(), ScriptError> {
let directories = &output.build_configuration.directories;

let vars = vars(output, state);
let mut s = String::new();
for v in vars {
shell_type.set_env_var(&mut s, &v.0, &v.1)?;
}

let platform = output.build_configuration.target_platform;

let additional_os_vars = os_vars(&directories.host_prefix, &platform);

for (k, v) in additional_os_vars {
shell_type.set_env_var(&mut s, &k, &v)?;
}

for (k, v) in output.recipe.build().script().env() {
shell_type.set_env_var(&mut s, k, v)?;
}

if !output.recipe.build().script().secrets().is_empty() {
tracing::error!("Secrets are not supported yet");
}

writeln!(out, "{}", s)?;

let host_prefix_activator = Activator::from_path(
&directories.host_prefix,
shell_type.clone(),
output.build_configuration.build_platform,
)?;
let current_path = std::env::var("PATH")
.ok()
.map(|p| std::env::split_paths(&p).collect::<Vec<_>>());

// if we are in a conda environment, we need to deactivate it before activating the host / build prefix
let conda_prefix = std::env::var("CONDA_PREFIX").ok().map(|p| p.into());

let activation_vars = ActivationVariables {
conda_prefix,
path: current_path,
path_modification_behavior: Default::default(),
};

let host_activation = host_prefix_activator
.activation(activation_vars)
.expect("Could not activate host prefix");

let build_prefix_activator = Activator::from_path(
&directories.build_prefix,
shell_type,
output.build_configuration.build_platform,
)?;

// We use the previous PATH and _no_ CONDA_PREFIX to stack the build
// prefix on top of the host prefix
let activation_vars = ActivationVariables {
conda_prefix: None,
path: Some(host_activation.path.clone()),
path_modification_behavior: Default::default(),
};

let build_activation = build_prefix_activator
.activation(activation_vars)
.expect("Could not activate host prefix");

writeln!(out, "{}", host_activation.script)?;
writeln!(out, "{}", build_activation.script)?;

Ok(())
}
1 change: 1 addition & 0 deletions src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ pub mod package_test;
pub mod packaging;
pub mod recipe;
pub mod render;
pub mod script;
pub mod selectors;
pub mod source;
pub mod system_tools;
Expand Down
Loading

0 comments on commit d77e887

Please sign in to comment.