Skip to content

feat: allow standalone benchmark processes #68

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 2 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 11 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -38,13 +38,17 @@ Refer to the [releases page](https://github.com/CodSpeedHQ/runner/releases) to s
> [!NOTE]
> For now, the CLI only supports Ubuntu 20.04, 22.04, 24.04 and Debian 11, 12.

### Authentication

First, authenticate with your CodSpeed account:

```bash
codspeed auth login
```

Then, run benchmarks with the following command:
### Running benchmarks

Run benchmarks with the following command (with a benchmark harness of your choice):

```bash
codspeed run <my-benchmark-command>
Expand All @@ -59,7 +63,12 @@ codspeed run pytest ./tests --codspeed
codspeed run pnpm vitest bench
```

It's also possible to run standalone benchmark processes:

```bash
codspeed run --standalone curl https://example.com
```

Usage: codspeed run [OPTIONS] [COMMAND]...

Arguments:
Expand All @@ -82,4 +91,4 @@ Use the `CODSPEED_LOG` environment variable to set the logging level:

```bash
CODSPEED_LOG=debug codspeed run ...
```
````
4 changes: 4 additions & 0 deletions codspeed.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
benchmarks:
- name: random bench
command: echo "hello world"
harnessed: false
7 changes: 7 additions & 0 deletions src/run/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ pub struct Config {
pub upload_url: Url,
pub token: Option<String>,
pub working_directory: Option<String>,

pub standalone: bool,
pub command: String,

pub instruments: Instruments,
Expand All @@ -35,6 +37,7 @@ impl Config {
instruments: Instruments::test(),
skip_upload: false,
skip_setup: false,
standalone: false,
}
}
}
Expand All @@ -50,6 +53,7 @@ impl TryFrom<RunArgs> for Config {
.map_err(|e| anyhow!("Invalid upload URL: {}, {}", raw_upload_url, e))?;
Ok(Self {
upload_url,
standalone: args.standalone,
token: args.token,
working_directory: args.working_directory,
instruments,
Expand Down Expand Up @@ -77,6 +81,7 @@ mod tests {
skip_upload: false,
skip_setup: false,
command: vec!["cargo".into(), "codspeed".into(), "bench".into()],
standalone: false,
})
.unwrap();
assert_eq!(config.upload_url, Url::parse(DEFAULT_UPLOAD_URL).unwrap());
Expand All @@ -86,6 +91,7 @@ mod tests {
assert!(!config.skip_upload);
assert!(!config.skip_setup);
assert_eq!(config.command, "cargo codspeed bench");
assert!(!config.standalone);
}

#[test]
Expand All @@ -99,6 +105,7 @@ mod tests {
skip_upload: true,
skip_setup: true,
command: vec!["cargo".into(), "codspeed".into(), "bench".into()],
standalone: true,
})
.unwrap();

Expand Down
5 changes: 5 additions & 0 deletions src/run/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,10 @@ fn show_banner() {

#[derive(Args, Debug)]
pub struct RunArgs {
/// Is the benchmark command a standalone command (i.e. not using a benchmarking framework)
#[arg(long, default_value = "false")]
pub standalone: bool,

/// The upload URL to use for uploading the results, useful for on-premises installations
#[arg(long)]
pub upload_url: Option<String>,
Expand Down Expand Up @@ -83,6 +87,7 @@ impl RunArgs {
/// Constructs a new `RunArgs` with default values for testing purposes
pub fn test() -> Self {
Self {
standalone: false,
upload_url: None,
token: None,
working_directory: None,
Expand Down
7 changes: 5 additions & 2 deletions src/run/runner/wall_time/executor.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ use crate::run::runner::helpers::run_command_with_log_pipe::run_command_with_log
use crate::run::runner::{ExecutorName, RunData, RunnerMode};
use crate::run::{check_system::SystemInfo, config::Config};
use async_trait::async_trait;
use std::env::consts::ARCH;
use std::fs::canonicalize;
use std::process::Command;

Expand All @@ -26,7 +27,8 @@ impl Executor for WallTimeExecutor {
run_data: &RunData,
_mongo_tracer: &Option<MongoTracer>,
) -> Result<()> {
let mut cmd = Command::new("sh");
let mut cmd = Command::new("setarch");
cmd.arg(ARCH).arg("-R");

cmd.envs(get_base_injected_env(
RunnerMode::WallTime,
Expand All @@ -38,7 +40,8 @@ impl Executor for WallTimeExecutor {
cmd.current_dir(abs_cwd);
}

cmd.args(["-c", get_bench_command(config)?.as_str()]);
// Configure perf
cmd.args(["sh", "-c", get_bench_command(config)?.as_str()]);

debug!("cmd: {:?}", cmd);
let status = run_command_with_log_pipe(cmd)
Expand Down