Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 10 additions & 29 deletions collector/src/compile/execute/bencher.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ use crate::utils::git::get_rustc_perf_commit;
use crate::{CollectorCtx, SelfProfileStorage};
use database::CollectionId;
use futures::stream::FuturesUnordered;
use futures::{future, StreamExt};
use futures::StreamExt;
use std::future::Future;
use std::pin::Pin;
use std::process::Command;
Expand All @@ -23,9 +23,10 @@ use std::{env, process};
use tokio::task::JoinSet;

pub struct RecordedSelfProfile {
collection: CollectionId,
scenario: database::Scenario,
profile: database::Profile,
codegen_backend: database::CodegenBackend,
target: database::Target,
files: SelfProfileFiles,
}

Expand Down Expand Up @@ -187,23 +188,17 @@ impl Processor for BenchProcessor<'_> {
database::Scenario::IncrementalPatch(patch.name)
}
};
let profile = match data.profile {
Profile::Check => database::Profile::Check,
Profile::Debug => database::Profile::Debug,
Profile::Doc => database::Profile::Doc,
Profile::DocJson => database::Profile::DocJson,
Profile::Opt => database::Profile::Opt,
Profile::Clippy => database::Profile::Clippy,
};
let profile: database::Profile = data.profile.into();

let version = get_rustc_perf_commit();
let collection = self.conn.collection_id(&version).await;

if let Some(files) = res.2 {
self.self_profiles.push(RecordedSelfProfile {
collection,
scenario,
profile,
codegen_backend: data.backend.into(),
target: data.target.into(),
files,
});

Expand Down Expand Up @@ -253,21 +248,6 @@ impl Processor for BenchProcessor<'_> {
fn postprocess_results<'b>(&'b mut self) -> Pin<Box<dyn Future<Output = ()> + 'b>> {
Box::pin(async move {
if let Some(self_profile_storage) = self.self_profile_storage {
let futs = self
.self_profiles
.iter()
.map(|profile| {
self.conn.record_raw_self_profile(
profile.collection,
self.collector_ctx.artifact_row_id,
self.benchmark.0.as_str(),
profile.profile,
profile.scenario,
)
})
.collect::<Vec<_>>();
future::join_all(futs).await;

let start = Instant::now();
let profile_count = self.self_profiles.len();

Expand All @@ -280,12 +260,13 @@ impl Processor for BenchProcessor<'_> {
}
}

let id = SelfProfileId {
artifact_id_number: self.collector_ctx.artifact_row_id,
collection: profile.collection,
let id = SelfProfileId::Simple {
artifact_id: self.collector_ctx.artifact_id.clone(),
benchmark: self.benchmark.clone(),
profile: profile.profile,
scenario: profile.scenario,
backend: profile.codegen_backend,
target: profile.target,
};
futures.spawn(self_profile_storage.store(id, profile.files));
}
Expand Down
2 changes: 2 additions & 0 deletions collector/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -361,6 +361,7 @@ impl CollectorStepBuilder {

CollectorCtx {
artifact_row_id,
artifact_id: artifact_id.clone(),
measured_compile_test_cases,
job_id: self.job_id,
}
Expand All @@ -370,6 +371,7 @@ impl CollectorStepBuilder {
/// Represents an in-progress run for a given artifact.
pub struct CollectorCtx {
pub artifact_row_id: ArtifactIdNumber,
pub artifact_id: ArtifactId,
/// Which tests cases were already computed **before** this collection began?
pub measured_compile_test_cases: HashSet<CompileTestCase>,
pub job_id: Option<u32>,
Expand Down
104 changes: 71 additions & 33 deletions collector/src/self_profile.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,33 +2,81 @@ use crate::compile::benchmark::BenchmarkName;
use crate::compile::execute::SelfProfileFiles;
use analyzeme::ProfilingData;
use anyhow::Context;
use database::{ArtifactIdNumber, CollectionId, Profile, Scenario};
use database::{
ArtifactId, ArtifactIdNumber, CodegenBackend, CollectionId, Profile, Scenario, Target,
};
use reqwest::StatusCode;
use std::future::Future;
use std::io::{Cursor, Read};
use std::path::{Path, PathBuf};
use std::pin::Pin;
use std::time::Instant;

// TODO: Record codegen backend in the self profile name
// TODO: remove collection ID from self-profile ID
/// Uniquely identifies a self-profile archive.
#[derive(Debug, Hash, PartialEq, Eq, Clone)]
pub struct SelfProfileId {
pub artifact_id_number: ArtifactIdNumber,
pub collection: CollectionId,
pub benchmark: BenchmarkName,
pub profile: Profile,
pub scenario: Scenario,
pub enum SelfProfileId {
/// Legacy ID with artifact ID number and collection ID
/// Exists for backwards compatibility with old self-profile links.
/// TODO: remove in Q3 2026
Legacy {
artifact_id_number: ArtifactIdNumber,
collection: CollectionId,
benchmark: BenchmarkName,
profile: Profile,
scenario: Scenario,
},
/// Simplified ID with artifact name and without collection ID
Simple {
artifact_id: ArtifactId,
benchmark: BenchmarkName,
profile: Profile,
scenario: Scenario,
backend: CodegenBackend,
target: Target,
},
}

impl SelfProfileId {
fn file_prefix(&self) -> PathBuf {
PathBuf::from("self-profile")
.join(self.artifact_id_number.0.to_string())
.join(self.benchmark.0.as_str())
.join(self.profile.to_string())
.join(self.scenario.to_id())
fn relative_file_path(&self) -> PathBuf {
match self {
// self-profile/<artifact id number>/<benchmark>/<profile>/<scenario>
// /self-profile-<collection-id>.mm_profdata.sz
SelfProfileId::Legacy {
profile,
benchmark,
collection,
artifact_id_number,
scenario,
} => PathBuf::from("self-profile")
.join(artifact_id_number.0.to_string())
.join(benchmark.0.as_str())
.join(profile.to_string())
.join(scenario.to_id())
.join(format!("self-profile-{collection}.mm_profdata.sz")),
// self-profile/<artifact id>/<benchmark>/<target>/<backend>/<profile>/<scenario>
// /self-profile.mm_profdata.sz
SelfProfileId::Simple {
artifact_id,
benchmark,
profile,
scenario,
backend: codegen_backend,
target,
} => {
let artifact_name = match artifact_id {
ArtifactId::Commit(c) => &c.sha,
ArtifactId::Tag(name) => name,
};
PathBuf::from("self-profile")
.join(artifact_name)
.join(benchmark.0.as_str())
.join(target.to_string())
.join(codegen_backend.to_string())
.join(profile.to_string())
.join(scenario.to_id())
.join("self-profile.mm_profdata.sz")
}
}
}
}

Expand Down Expand Up @@ -84,10 +132,7 @@ impl LocalSelfProfileStorage {
}

fn path(&self, id: &SelfProfileId) -> PathBuf {
let prefix = id.file_prefix();
self.directory
.join(prefix)
.join(format!("self-profile-{}.mm_profdata.sz", id.collection))
self.directory.join(id.relative_file_path())
}
}

Expand Down Expand Up @@ -136,23 +181,16 @@ impl S3SelfProfileStorage {
}
}

fn s3_self_profile_filename(id: &SelfProfileId) -> String {
format!("self-profile-{}.mm_profdata.sz", id.collection)
}

impl SelfProfileStorage for S3SelfProfileStorage {
fn store(
&self,
id: SelfProfileId,
files: SelfProfileFiles,
) -> Pin<Box<dyn Future<Output = anyhow::Result<()>> + Send>> {
Box::pin(async move {
// Files are placed at
// * self-profile/<artifact id>/<benchmark>/<profile>/<scenario>
// /self-profile-<collection-id>.{extension}
let prefix = id.file_prefix();
let file_path = id.relative_file_path();
let upload = tempfile::NamedTempFile::new().context("cannot create temporary file")?;
let filename = match files {
match files {
SelfProfileFiles::Eight { file } => {
let data = tokio::fs::read(file)
.await
Expand All @@ -164,12 +202,10 @@ impl SelfProfileStorage for S3SelfProfileStorage {
tokio::fs::write(upload.path(), &compressed)
.await
.context("cannot write compressed self-profile data")?;

s3_self_profile_filename(&id)
}
};

log::info!("Uploading self-profile to {}/{filename}", prefix.display());
log::info!("Uploading self-profile to {}", file_path.display());

let output = tokio::process::Command::new("aws")
.arg("s3")
Expand All @@ -180,7 +216,9 @@ impl SelfProfileStorage for S3SelfProfileStorage {
.arg(upload.path())
.arg(format!(
"s3://rustc-perf/{}",
&prefix.join(filename).to_str().unwrap()
file_path
.to_str()
.expect("Non UTF-8 path used for self-profile")
))
.spawn()
.context("cannot spawn aws binary")?
Expand All @@ -202,7 +240,7 @@ impl SelfProfileStorage for S3SelfProfileStorage {
&self,
id: SelfProfileId,
) -> Pin<Box<dyn Future<Output = anyhow::Result<Option<Vec<u8>>>> + Send>> {
let path = id.file_prefix().join(s3_self_profile_filename(&id));
let path = id.relative_file_path();
let url = format!(
"https://perf-data.rust-lang.org/{}",
path.to_str().expect("Non UTF-8 path used for self-profile")
Expand Down
12 changes: 0 additions & 12 deletions database/src/pool.rs
Original file line number Diff line number Diff line change
Expand Up @@ -71,18 +71,6 @@ pub trait Connection: Send + Sync {
target: Target,
value: f64,
);
/// Records a self-profile artifact in S3.
///
/// The upload is a separate step (which may fail or be canceled, but that's
/// fine, the database just stores that the files *may* be there).
async fn record_raw_self_profile(
&self,
collection: CollectionId,
artifact: ArtifactIdNumber,
benchmark: &str,
profile: Profile,
scenario: Scenario,
);
async fn record_error(
&self,
artifact: ArtifactIdNumber,
Expand Down
15 changes: 0 additions & 15 deletions database/src/pool/postgres.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1263,21 +1263,6 @@ where
.unwrap()
.map(|r| r.get::<_, i32>(0) as u32)
}
async fn record_raw_self_profile(
&self,
collection: CollectionId,
artifact: ArtifactIdNumber,
benchmark: &str,
profile: Profile,
scenario: Scenario,
) {
let profile = profile.to_string();
let scenario = scenario.to_string();
self.conn().execute(
"insert into raw_self_profile (aid, cid, crate, profile, cache) VALUES ($1, $2, $3, $4, $5)",
&[&(artifact.0 as i32), &collection.0, &benchmark, &profile, &scenario],
).await.unwrap();
}
async fn list_self_profile(
&self,
aid: ArtifactId,
Expand Down
9 changes: 0 additions & 9 deletions database/src/pool/sqlite.rs
Original file line number Diff line number Diff line change
Expand Up @@ -784,15 +784,6 @@ impl Connection for SqliteConnection {
)
.unwrap();
}
async fn record_raw_self_profile(
&self,
_collection: CollectionId,
_artifact: ArtifactIdNumber,
_benchmark: &str,
_profile: Profile,
_scenario: crate::Scenario,
) {
}

async fn record_error(
&self,
Expand Down
4 changes: 3 additions & 1 deletion site/frontend/src/components/perfetto-link.vue
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,9 @@ const link = computed(() => {
props.artifact.commit,
`${props.testCase.benchmark}`,
props.testCase.scenario,
props.testCase.profile.toLowerCase()
props.testCase.profile.toLowerCase(),
props.testCase.backend,
props.testCase.target
);
});

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ function createSectionsSelector(): CompileDetailSectionsSelector {
benchmark: props.testCase.benchmark,
profile: props.testCase.profile,
scenario: props.testCase.scenario,
backend: props.testCase.backend,
target: props.testCase.target,
start: props.baseArtifact.commit,
end: props.artifact.commit,
};
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ export interface CompileDetailSectionsSelector {
benchmark: string;
scenario: string;
profile: string;
backend: string;
target: string;
}

export interface CompileDetailSections {
Expand Down Expand Up @@ -92,7 +94,7 @@ export const COMPILE_DETAIL_SECTIONS_RESOLVER: CachedDataLoader<
CompileDetailSections
> = new CachedDataLoader(
(key: CompileDetailSectionsSelector) =>
`${key.benchmark};${key.profile};${key.scenario};${key.start};${key.end}`,
`${key.benchmark};${key.profile};${key.scenario};${key.backend};${key.target};${key.start};${key.end}`,
loadSectionsDetail
);

Expand All @@ -105,6 +107,8 @@ async function loadSectionsDetail(
benchmark: selector.benchmark,
scenario: selector.scenario,
profile: selector.profile,
backend: selector.backend,
target: selector.target,
};
return await getJson<CompileDetailSections>(
COMPARE_COMPILE_DETAIL_SECTIONS_DATA_URL,
Expand Down
Loading