From f982e37fe5f64aa4b82f07ce622d6fb0c7ea23ce Mon Sep 17 00:00:00 2001 From: Lucy Hochkamp Date: Fri, 10 Oct 2025 17:40:25 +0200 Subject: [PATCH] add attic push functionality --- .envrc | 2 +- .gitignore | 1 + flake.lock | 23 ++++++++++++- flake.nix | 95 +++++++++++++++++++++++++++++++++++---------------- shell.nix | 4 --- src/config.rs | 3 ++ src/copy.rs | 91 ++++++++++++++++++++++++++++++++++++------------ src/main.rs | 14 ++++++++ 8 files changed, 175 insertions(+), 58 deletions(-) delete mode 100644 shell.nix diff --git a/.envrc b/.envrc index 1d953f4..3550a30 100644 --- a/.envrc +++ b/.envrc @@ -1 +1 @@ -use nix +use flake diff --git a/.gitignore b/.gitignore index d787b70..f717ddd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ /target /result +.direnv diff --git a/flake.lock b/flake.lock index 7278b9c..863b63d 100644 --- a/flake.lock +++ b/flake.lock @@ -18,7 +18,28 @@ }, "root": { "inputs": { - "nixpkgs": "nixpkgs" + "nixpkgs": "nixpkgs", + "rust-overlay": "rust-overlay" + } + }, + "rust-overlay": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1760063676, + "narHash": "sha256-s5Fjh43skH2L+avOGioLmEHoYZffDbg3abV5h0gjeew=", + "owner": "oxalica", + "repo": "rust-overlay", + "rev": "897deed0923cc5a1d560c5176abe0d172ec9716d", + "type": "github" + }, + "original": { + "owner": "oxalica", + "repo": "rust-overlay", + "type": "github" } } }, diff --git a/flake.nix b/flake.nix index 9b6009e..5fa7c48 100644 --- a/flake.nix +++ b/flake.nix @@ -1,34 +1,47 @@ { inputs = { nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable"; + rust-overlay = { + url = "github:oxalica/rust-overlay"; + inputs.nixpkgs.follows = "nixpkgs"; + }; }; - outputs = { - nixpkgs, - self, - }: let - forAllSystems = fn: - builtins.foldl' (attrs: system: let - outputs = fn system; - outputNames = builtins.attrNames outputs; - in - builtins.foldl' (attrs: outName: let - existing = attrs.${outName} or {}; - new = existing // {${system} = outputs.${outName};}; - in - attrs // {${outName} = new;}) - attrs - outputNames) {} ["x86_64-linux" "aarch64-linux"]; - in + outputs = + { nixpkgs + , rust-overlay + , self + , + }: + let + forAllSystems = fn: + builtins.foldl' + (attrs: system: + let + outputs = fn system; + outputNames = builtins.attrNames outputs; + in + builtins.foldl' + (attrs: outName: + let + existing = attrs.${outName} or { }; + new = existing // { ${system} = outputs.${outName}; }; + in + attrs // { ${outName} = new; }) + attrs + outputNames) + { } [ "x86_64-linux" "aarch64-linux" ]; + in { overlays.default = final: prev: { - nix-ci = final.lib.flip final.callPackage {} ({ - rustPlatform, - lib, - makeWrapper, - nix-eval-jobs, - }: let - cargoToml = builtins.fromTOML (builtins.readFile ./Cargo.toml); - in + nix-ci = final.lib.flip final.callPackage { } ({ rustPlatform + , lib + , makeWrapper + , nix-eval-jobs + , + }: + let + cargoToml = builtins.fromTOML (builtins.readFile ./Cargo.toml); + in rustPlatform.buildRustPackage { pname = cargoToml.package.name; version = cargoToml.package.version; @@ -40,7 +53,7 @@ ./Cargo.lock ]; }; - nativeBuildInputs = [makeWrapper]; + nativeBuildInputs = [ makeWrapper ]; cargoLock.lockFile = ./Cargo.lock; preFixup = '' wrapProgram "$out/bin/nix-ci" \ @@ -49,13 +62,35 @@ }); }; } - // forAllSystems (system: let + // forAllSystems (system: + let pkgs = import nixpkgs { inherit system; - overlays = [self.overlays.default]; + overlays = [ + self.overlays.default + rust-overlay.overlays.default + + + (final: prev: { + inherit (prev.lixPackageSets.stable) + nixpkgs-review + nix-eval-jobs + nix-fast-build + colmena; + }) + ]; + }; + rust = pkgs.rust-bin.stable.latest.default.override { + extensions = [ + "rust-src" # for rust-analyzer + "rust-analyzer" + ]; + }; + in + { + devShells.default = pkgs.mkShell { + nativeBuildInputs = [ pkgs.nix-eval-jobs rust pkgs.attic-client ]; }; - in { - devShells.default = import ./shell.nix {inherit pkgs;}; packages = { default = pkgs.nix-ci; nix-ci = pkgs.nix-ci; diff --git a/shell.nix b/shell.nix deleted file mode 100644 index cbf60d9..0000000 --- a/shell.nix +++ /dev/null @@ -1,4 +0,0 @@ -{pkgs ? import {}}: -pkgs.mkShell { - nativeBuildInputs = with pkgs; [nix-eval-jobs]; -} diff --git a/src/config.rs b/src/config.rs index 7ad4bc3..4b178af 100644 --- a/src/config.rs +++ b/src/config.rs @@ -17,6 +17,9 @@ pub struct Options { #[arg(long)] /// store uri where build outputs will be uploaded to pub copy_to: Option, + #[arg(long)] + /// attic store where build outputs will be uploaded to + pub copy_to_attic: Option, #[arg(long, default_value = "4")] /// maximum number of evaluation workers diff --git a/src/copy.rs b/src/copy.rs index 9a87de1..68fca08 100644 --- a/src/copy.rs +++ b/src/copy.rs @@ -51,34 +51,57 @@ pub async fn copy_loop( continue; } }; - let Some(copy_to) = &opts.copy_to else { - continue; - }; for store_path in &valid_paths { if paths_copied.contains(store_path) { continue; } paths_copied.push(store_path.clone()); - match copy_path(store_path, copy_to).await { - Ok(()) => { - let _ = result_tx - .send(NixCiResult { - r#type: NixCiResultType::Copy, - path: store_path.clone(), - success: true, - }) - .await; - tracing::info!("copied path {}", store_path); + if let Some(copy_to) = &opts.copy_to { + match copy_path(store_path, copy_to).await { + Ok(()) => { + let _ = result_tx + .send(NixCiResult { + r#type: NixCiResultType::Copy, + path: store_path.clone(), + success: true, + }) + .await; + tracing::info!("copied path {}", store_path); + } + Err(e) => { + let _ = result_tx + .send(NixCiResult { + r#type: NixCiResultType::Copy, + path: store_path.clone(), + success: false, + }) + .await; + tracing::error!("failed to copy path {}: {}", store_path, e); + } } - Err(e) => { - let _ = result_tx - .send(NixCiResult { - r#type: NixCiResultType::Copy, - path: store_path.clone(), - success: false, - }) - .await; - tracing::error!("failed to copy path {}: {}", store_path, e); + } + if let Some(copy_to_attic) = &opts.copy_to_attic { + match copy_path_attic(store_path, copy_to_attic).await { + Ok(()) => { + let _ = result_tx + .send(NixCiResult { + r#type: NixCiResultType::Copy, + path: store_path.clone(), + success: true, + }) + .await; + tracing::info!("copied path {}", store_path); + } + Err(e) => { + let _ = result_tx + .send(NixCiResult { + r#type: NixCiResultType::Copy, + path: store_path.clone(), + success: false, + }) + .await; + tracing::error!("failed to copy path {}: {}", store_path, e); + } } } } @@ -141,6 +164,30 @@ async fn check_path_validity(store_path: &str) -> anyhow::Result { Ok(cmd.status.success()) } +async fn copy_path_attic(store_path: &str, attic_store: &str) -> anyhow::Result<()> { + let mut cmd = WrappedChild::new( + Command::new("attic").args(&["push", attic_store, store_path]), + Some(format!("attic push {} {}", attic_store, store_path)), + )?; + loop { + match cmd.next_line().await? { + ChildOutput::Finished => break, + ChildOutput::Stderr(line) | ChildOutput::Stdout(line) => { + tracing::info!("{}", line); + } + } + } + let exit_status = cmd.exit_status().unwrap(); + if exit_status.success() { + Ok(()) + } else { + Err(anyhow::anyhow!( + "nix copy exited with non-success {}", + exit_status + )) + } +} + async fn copy_path(store_path: &str, copy_to: &str) -> anyhow::Result<()> { let mut cmd = WrappedChild::new( Command::new("nix").args(&["copy", "--to", copy_to, store_path]), diff --git a/src/main.rs b/src/main.rs index af7a05b..2c3038b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -63,6 +63,20 @@ async fn main() -> anyhow::Result<()> { std::process::exit(1); } } + if let Some(store) = &opts.copy_to_attic { + let cmd = Command::new("attic") + .args(&["cache", "info", store]) + .output() + .await?; + if !cmd.status.success() { + tracing::error!( + "{:?} is not a valid attic store: {}", + store, + String::from_utf8_lossy(&cmd.stderr), + ); + std::process::exit(1); + } + } tracing::debug!("running with options {:?}", opts); let opts = Arc::new(opts);