Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Eagerly load self-profile files #1798

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Eagerly load self-profile files #1798

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
19a76cf
Revert "Merge pull request #1797 from Kobzol/self-profile-unique-dir"
Kobzol Jan 9, 2024
ec3d464
Eagerly load self-profile files into memory
Kobzol Jan 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 3 additions & 17 deletions collector/src/bin/rustc-fake.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ use std::ffi::OsString;
use std::fs;
use std::path::PathBuf;
use std::process::Command;
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use std::time::{Duration, Instant};

fn determinism_env(cmd: &mut Command) {
// Since rust-lang/rust#89836, rustc stable crate IDs include a hash of the
Expand Down Expand Up @@ -35,20 +35,6 @@ fn run_with_determinism_env(mut cmd: Command) {
);
}

// We want each rustc execution to have a separate self-profile directory,
// to avoid overwriting the results. PID of this process and timestamp should
// hopefully be unique enough.
fn create_self_profile_dir() -> PathBuf {
let pid = std::process::id();
let timestamp = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_millis();
let name = format!("self-profile-output-{pid}-{timestamp}");

std::env::current_dir().unwrap().join(name)
}

fn main() {
let mut args_os = env::args_os();
let name = args_os.next().unwrap().into_string().unwrap();
Expand Down Expand Up @@ -116,7 +102,7 @@ fn main() {
.arg(&tool)
.args(&args);

let prof_out_dir = create_self_profile_dir();
let prof_out_dir = std::env::current_dir().unwrap().join("self-profile-output");
if wrapper == "PerfStatSelfProfile" {
cmd.arg(&format!(
"-Zself-profile={}",
Expand Down Expand Up @@ -187,7 +173,7 @@ fn main() {
let mut tool = Command::new(tool);
tool.args(&args);

let prof_out_dir = create_self_profile_dir();
let prof_out_dir = std::env::current_dir().unwrap().join("self-profile-output");
if wrapper == "XperfStatSelfProfile" {
tool.arg(&format!(
"-Zself-profile={}",
Expand Down
32 changes: 15 additions & 17 deletions collector/src/compile/execute/bencher.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ use crate::compile::benchmark::scenario::Scenario;
use crate::compile::benchmark::BenchmarkName;
use crate::compile::execute;
use crate::compile::execute::{
rustc, DeserializeStatError, PerfTool, ProcessOutputData, Processor, Retry, SelfProfileFiles,
rustc, DeserializeStatError, LoadedSelfProfile, PerfTool, ProcessOutputData, Processor, Retry,
Stats,
};
use crate::toolchain::Toolchain;
Expand All @@ -25,7 +25,7 @@ pub struct RecordedSelfProfile {
collection: CollectionId,
scenario: database::Scenario,
profile: database::Profile,
files: SelfProfileFiles,
self_profile: LoadedSelfProfile,
}

// Tools usable with the benchmarking subcommands.
Expand Down Expand Up @@ -187,12 +187,12 @@ impl<'a> Processor for BenchProcessor<'a> {
// If the gathered metrics were produced with self profile enabled, then they
// are not realistic. Do not store the metrics into the DB for self-profile
// runs to avoid unnecessary DB storage.
if let Some(files) = res.2 {
if let Some(self_profile) = res.1 {
self.self_profiles.push(RecordedSelfProfile {
collection,
scenario,
profile,
files,
self_profile,
});
} else {
self.insert_stats(collection, scenario, profile, data.backend, res.0)
Expand Down Expand Up @@ -256,8 +256,11 @@ impl<'a> Processor for BenchProcessor<'a> {
.join(self.benchmark.0.as_str())
.join(profile.profile.to_string())
.join(profile.scenario.to_id());
let upload =
SelfProfileS3Upload::new(prefix, profile.collection, profile.files);
let upload = SelfProfileS3Upload::new(
prefix,
profile.collection,
&profile.self_profile.raw_profile_data,
);
uploads.push_back(upload);
}
for upload in uploads {
Expand All @@ -275,25 +278,20 @@ impl SelfProfileS3Upload {
fn new(
prefix: PathBuf,
collection: database::CollectionId,
files: SelfProfileFiles,
profile_data: &[u8],
) -> SelfProfileS3Upload {
// Files are placed at
// * self-profile/<artifact id>/<benchmark>/<profile>/<scenario>
// /self-profile-<collection-id>.{extension}
let upload = tempfile::NamedTempFile::new()
.context("create temporary file")
.unwrap();
let filename = match files {
SelfProfileFiles::Eight { file } => {
let data = std::fs::read(file).expect("read profile data");
let mut data = snap::read::FrameEncoder::new(&data[..]);
let mut compressed = Vec::new();
data.read_to_end(&mut compressed).expect("compressed");
std::fs::write(upload.path(), &compressed).expect("write compressed profile data");
let mut data = snap::read::FrameEncoder::new(profile_data);
let mut compressed = Vec::new();
data.read_to_end(&mut compressed).expect("compressed");
std::fs::write(upload.path(), &compressed).expect("write compressed profile data");

format!("self-profile-{}.mm_profdata.sz", collection)
}
};
let filename = format!("self-profile-{}.mm_profdata.sz", collection);

let child = Command::new("aws")
.arg("s3")
Expand Down
36 changes: 17 additions & 19 deletions collector/src/compile/execute/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -464,7 +464,7 @@ fn store_documentation_size_into_stats(stats: &mut Stats, doc_dir: &Path) {
}
}

fn store_artifact_sizes_into_stats(stats: &mut Stats, profile: &SelfProfile) {
fn store_artifact_sizes_into_stats(stats: &mut Stats, profile: &LoadedSelfProfile) {
for artifact in profile.artifact_sizes.iter() {
stats
.stats
Expand All @@ -484,13 +484,9 @@ enum DeserializeStatError {
IOError(#[from] std::io::Error),
}

enum SelfProfileFiles {
Eight { file: PathBuf },
}

fn process_stat_output(
output: process::Output,
) -> Result<(Stats, Option<SelfProfile>, Option<SelfProfileFiles>), DeserializeStatError> {
) -> Result<(Stats, Option<LoadedSelfProfile>), DeserializeStatError> {
let stdout = String::from_utf8(output.stdout.clone()).expect("utf8 output");
let mut stats = Stats::new();

Expand Down Expand Up @@ -569,11 +565,11 @@ fn process_stat_output(
if stats.is_empty() {
return Err(DeserializeStatError::NoOutput(output));
}
let (profile, files) = match (self_profile_dir, self_profile_crate) {
let profile = match (self_profile_dir, self_profile_crate) {
(Some(dir), Some(krate)) => parse_self_profile(dir, krate)?,
_ => (None, None),
_ => None,
};
Ok((stats, profile, files))
Ok((stats, profile))
}

#[derive(Clone)]
Expand Down Expand Up @@ -608,14 +604,16 @@ impl Stats {
}

#[derive(serde::Deserialize, Clone)]
pub struct SelfProfile {
pub struct LoadedSelfProfile {
pub artifact_sizes: Vec<ArtifactSize>,
// Data of the profile loaded into memory
pub raw_profile_data: Vec<u8>,
}

fn parse_self_profile(
dir: PathBuf,
crate_name: String,
) -> std::io::Result<(Option<SelfProfile>, Option<SelfProfileFiles>)> {
) -> std::io::Result<Option<LoadedSelfProfile>> {
// First, find the `.mm_profdata` file with the self-profile data.
let mut full_path = None;
// We don't know the pid of rustc, and can't easily get it -- we only know the
Expand All @@ -629,24 +627,24 @@ fn parse_self_profile(
break;
}
}
let (profile, files) = if let Some(profile_path) = full_path {
let profile = if let Some(profile_path) = full_path {
// measureme 0.8+ uses a single file
let data = fs::read(&profile_path)?;
let results = analyzeme::ProfilingData::from_paged_buffer(data, None)
let raw_profile_data = fs::read(&profile_path)?;
let results = analyzeme::ProfilingData::from_paged_buffer(raw_profile_data.clone(), None)
.map_err(|error| {
eprintln!("Cannot read self-profile data: {error:?}");
std::io::Error::new(ErrorKind::InvalidData, error)
})?
.perform_analysis();
let profile = SelfProfile {
let profile = LoadedSelfProfile {
artifact_sizes: results.artifact_sizes,
raw_profile_data,
};
let files = SelfProfileFiles::Eight { file: profile_path };
(Some(profile), Some(files))
Some(profile)
} else {
// The old "3 files format" is not supported by analyzeme anymore, so we don't handle it
// here.
(None, None)
None
};
Ok((profile, files))
Ok(profile)
}
Loading