initial commit
This commit is contained in:
⛧-440729 [sophie]
commit
8e1813ca0d
14 changed files with 2005 additions and 0 deletions
300
src/build.rs
Normal file
300
src/build.rs
Normal file
|
|
@ -0,0 +1,300 @@
|
|||
use std::collections::{HashMap, VecDeque};
|
||||
|
||||
use tokio::{process::Command, sync::mpsc::Receiver};
|
||||
|
||||
use crate::{
|
||||
types::{
|
||||
NixInternalLogLine, NixInternalLogLineActivity, NixInternalLogLineActivityType,
|
||||
NixInternalLogLineResult, NixJob,
|
||||
},
|
||||
util::{ChildOutput, WrappedChild},
|
||||
};
|
||||
|
||||
pub enum BuildLoopMessage {
|
||||
Job(NixJob),
|
||||
Stop,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct ActivityData {
|
||||
#[allow(dead_code)]
|
||||
parent: Option<u64>,
|
||||
r#type: NixInternalLogLineActivityType,
|
||||
running: bool,
|
||||
data: Option<ActivityDataPerType>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
enum ActivityDataPerType {
|
||||
Build { drv_path: String, state: BuildState },
|
||||
Substitute { store_path: String },
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
enum BuildState {
|
||||
Started,
|
||||
Running,
|
||||
Done,
|
||||
Failed,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
struct BuildProgress {
|
||||
done: u64,
|
||||
running: u64,
|
||||
failed: u64,
|
||||
}
|
||||
|
||||
pub async fn build_loop(mut rx: Receiver<BuildLoopMessage>) {
|
||||
let mut paths_built = Vec::new();
|
||||
while let Some(msg) = rx.recv().await {
|
||||
let job = match msg {
|
||||
BuildLoopMessage::Job(job) => job,
|
||||
BuildLoopMessage::Stop => {
|
||||
tracing::debug!("stop signal received");
|
||||
break;
|
||||
}
|
||||
};
|
||||
tracing::info!("building {}", job.attr);
|
||||
if !paths_built.contains(&job.drv_path) {
|
||||
paths_built.push(job.drv_path.clone());
|
||||
if let Err(e) = run_build(job).await {
|
||||
tracing::error!("nix build process errored! {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(job), fields(attr = job.attr))]
|
||||
pub async fn run_build(job: NixJob) -> anyhow::Result<()> {
|
||||
let mut child = WrappedChild::new(
|
||||
Command::new("nix")
|
||||
.args(&[
|
||||
"build",
|
||||
"--keep-going",
|
||||
"--verbose",
|
||||
"--log-format",
|
||||
"internal-json",
|
||||
])
|
||||
.arg(format!("{}^*", job.drv_path)),
|
||||
Some(format!("nix build {}", job.attr)),
|
||||
)?;
|
||||
let mut activities = HashMap::<u64, ActivityData>::new();
|
||||
let mut progress = BuildProgress {
|
||||
done: 0,
|
||||
running: 0,
|
||||
failed: 0,
|
||||
};
|
||||
let mut started_builds = VecDeque::new();
|
||||
let mut reported_builds = VecDeque::new();
|
||||
loop {
|
||||
let line = match child.next_line().await? {
|
||||
ChildOutput::Finished => break,
|
||||
ChildOutput::Stderr(line) => line,
|
||||
ChildOutput::Stdout(line) => {
|
||||
tracing::info!(line);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
if !line.starts_with("@nix ") {
|
||||
tracing::warn!("invalid nix build output: {}", line);
|
||||
continue;
|
||||
}
|
||||
let log: NixInternalLogLine = match serde_json::from_str(&line[4..]) {
|
||||
Ok(log) => log,
|
||||
Err(e) => {
|
||||
tracing::warn!("failed to parse nix log output {:?}: {}", line, e);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
tracing::trace!("log line: {:?}", log);
|
||||
match log {
|
||||
NixInternalLogLine::Msg { msg, .. } => {
|
||||
tracing::info!("{}", msg);
|
||||
}
|
||||
NixInternalLogLine::Result { id, result } => {
|
||||
let activity = activities.get_mut(&id);
|
||||
if activity.is_none() {
|
||||
tracing::warn!("received result for unknown activity {}!", id);
|
||||
}
|
||||
match result {
|
||||
NixInternalLogLineResult::BuildLogLine { log }
|
||||
| NixInternalLogLineResult::PostBuildLogLine { log } => {
|
||||
tracing::info!("{}", log)
|
||||
}
|
||||
// apparently only reported for the realise activity, with activity types of
|
||||
// FileTransfer and CopyPath. contains bytes downloaded and unpacked, respectively
|
||||
NixInternalLogLineResult::SetExpected { .. } => {}
|
||||
// progress reporting, depends on the activity type
|
||||
NixInternalLogLineResult::Progress {
|
||||
done,
|
||||
expected: _,
|
||||
running,
|
||||
failed,
|
||||
} => {
|
||||
if let Some(activity) = activity {
|
||||
match activity.r#type {
|
||||
// meta-progress counting the derivations being built as part of the build
|
||||
NixInternalLogLineActivityType::Builds => {
|
||||
while running > progress.running {
|
||||
progress.running += 1;
|
||||
if let Some(build_activity_id) = started_builds.pop_front()
|
||||
{
|
||||
let Some(a) = activities.get_mut(&build_activity_id)
|
||||
else {
|
||||
panic!("this can't happen (id without activity)");
|
||||
};
|
||||
let ActivityDataPerType::Build { state, .. } =
|
||||
a.data.as_mut().unwrap()
|
||||
else {
|
||||
panic!(
|
||||
"this can't happen (build activity without build data)"
|
||||
);
|
||||
};
|
||||
*state = BuildState::Running;
|
||||
} else {
|
||||
tracing::warn!(
|
||||
"no build started but we got a running report?"
|
||||
);
|
||||
}
|
||||
}
|
||||
while failed > progress.failed {
|
||||
progress.failed += 1;
|
||||
reported_builds.push_back(BuildState::Failed);
|
||||
}
|
||||
while done > progress.done {
|
||||
progress.done += 1;
|
||||
reported_builds.push_back(BuildState::Done);
|
||||
}
|
||||
}
|
||||
// meta-progress counting the paths being downloaded as part of the build
|
||||
NixInternalLogLineActivityType::CopyPaths => {}
|
||||
// progress for a single download
|
||||
NixInternalLogLineActivityType::FileTransfer => {}
|
||||
// progress for a file unpack(?) downloaded nar -> store
|
||||
NixInternalLogLineActivityType::CopyPath => {}
|
||||
_ => {
|
||||
tracing::warn!(
|
||||
"progress reported for unhandled activity type {:?}",
|
||||
activity.r#type
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
tracing::warn!("unhandled result: activity {}, {:?}", id, result);
|
||||
}
|
||||
}
|
||||
}
|
||||
NixInternalLogLine::Start {
|
||||
id,
|
||||
level: _,
|
||||
text,
|
||||
parent,
|
||||
activity,
|
||||
} => {
|
||||
tracing::debug!(
|
||||
"activity {} (parent {}) started: {:?}",
|
||||
id,
|
||||
parent,
|
||||
activity
|
||||
);
|
||||
let mut entry = {
|
||||
let parent = if parent == 0 { None } else { Some(parent) };
|
||||
activities.entry(id).insert_entry(ActivityData {
|
||||
parent,
|
||||
r#type: activity.get_type(),
|
||||
running: true,
|
||||
data: None,
|
||||
})
|
||||
};
|
||||
if !text.is_empty() {
|
||||
tracing::info!("{}", text);
|
||||
}
|
||||
match activity {
|
||||
// these seem to be meta-activities for reporting progress
|
||||
NixInternalLogLineActivity::Builds => {}
|
||||
NixInternalLogLineActivity::CopyPaths => {}
|
||||
NixInternalLogLineActivity::Realise => {}
|
||||
// just ignore this, we don't really care about these
|
||||
NixInternalLogLineActivity::Unknown => {}
|
||||
NixInternalLogLineActivity::FileTransfer => {}
|
||||
NixInternalLogLineActivity::QueryPathInfo { .. } => {
|
||||
// has FileTransfer as child
|
||||
}
|
||||
NixInternalLogLineActivity::CopyPath => {
|
||||
// has FileTransfer as child (when downloading)
|
||||
}
|
||||
NixInternalLogLineActivity::Substitute { store_path, .. } => {
|
||||
// has CopyPath as child
|
||||
entry
|
||||
.get_mut()
|
||||
.data
|
||||
.replace(ActivityDataPerType::Substitute { store_path });
|
||||
}
|
||||
// useful activities
|
||||
NixInternalLogLineActivity::Build { drv_path, .. } => {
|
||||
entry.get_mut().data.replace(ActivityDataPerType::Build {
|
||||
drv_path,
|
||||
state: BuildState::Started,
|
||||
});
|
||||
started_builds.push_back(id);
|
||||
}
|
||||
_ => {
|
||||
tracing::warn!(
|
||||
"unhandled start: activity {} (parent {}), {:?}",
|
||||
id,
|
||||
parent,
|
||||
activity
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
NixInternalLogLine::Stop { id } => {
|
||||
tracing::debug!("activity {} stopped", id);
|
||||
if let Some(entry) = activities.get_mut(&id) {
|
||||
entry.running = false;
|
||||
match entry.r#type {
|
||||
NixInternalLogLineActivityType::Build => {
|
||||
let ActivityDataPerType::Build { state, .. } =
|
||||
entry.data.as_mut().unwrap()
|
||||
else {
|
||||
panic!("this can't happen (build activity without build data)");
|
||||
};
|
||||
if let Some(report) = reported_builds.pop_front() {
|
||||
*state = report;
|
||||
} else {
|
||||
tracing::warn!(
|
||||
"build stopped but no report (don't know whether it failed)"
|
||||
);
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
let exit_status = child
|
||||
.exit_status()
|
||||
.ok_or(anyhow::anyhow!("child has exited, but no exit status????"))?;
|
||||
if !exit_status.success() {
|
||||
tracing::warn!("nix build failed with {}!", exit_status);
|
||||
}
|
||||
for (_, a) in &activities {
|
||||
match a.data.as_ref() {
|
||||
Some(ActivityDataPerType::Build { drv_path, state }) => match state {
|
||||
BuildState::Done => tracing::info!("derivation {} built successfully", drv_path),
|
||||
BuildState::Failed => tracing::warn!("derivation {} failed to build", drv_path),
|
||||
_ => tracing::warn!("derivation {} somehow still running??", drv_path),
|
||||
},
|
||||
Some(ActivityDataPerType::Substitute { store_path }) => {
|
||||
tracing::info!("downloaded {}", store_path);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
// TODO: return the drv paths to upload!
|
||||
Ok(())
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue