Skip to content

cli: Add --nix-option to allow passing arbitrary nix options #126

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
zhaofengli merged 3 commits into from
Nov 14, 2022
Merged

cli: Add --nix-option to allow passing arbitrary nix options #126

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
19bf776
cli: Add --nix-option to allow passing arbitrary nix options
ThinkChaos Nov 10, 2022
d73fa5d
cli: Explicit `value_name` for --node to be all caps like others
ThinkChaos Nov 10, 2022
90c88dd
refactor: rename NixOptions to NixFlags
ThinkChaos Nov 10, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 13 additions & 2 deletions src/cli.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
use std::env;

use clap::{
builder::PossibleValue, value_parser, Arg, ArgMatches, ColorChoice, Command as ClapCommand,
ValueEnum,
builder::PossibleValue, value_parser, Arg, ArgAction, ArgMatches, ColorChoice,
Command as ClapCommand, ValueEnum,
};
use clap_complete::Shell;
use const_format::concatcp;
Expand Down Expand Up @@ -160,6 +160,17 @@ pub fn build_cli(include_internal: bool) -> ClapCommand {
.long_help("Passes --impure to Nix commands")
.global(true)
.num_args(0))
.arg(Arg::new("nix-option")
.long("nix-option")
.help("Passes an arbitrary option to Nix commands")
.long_help(r#"Passes arbitrary options to Nix commands

This only works when building locally.
"#)
.global(true)
.num_args(2)
.value_names(["NAME", "VALUE"])
.action(ArgAction::Append))
.arg(Arg::new("color")
.long("color")
.help("When to colorize the output")
Expand Down
3 changes: 2 additions & 1 deletion src/command/apply_local.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ By default, Colmena will deploy keys set in `deployment.keys` before activating
.num_args(0))
.arg(Arg::new("node")
.long("node")
.value_name("NODE")
.help("Override the node name to use")
.num_args(1))

Expand Down Expand Up @@ -104,7 +105,7 @@ pub async fn run(_global_args: &ArgMatches, local_args: &ArgMatches) -> Result<(

let target = {
if let Some(info) = hive.deployment_info_single(&hostname).await.unwrap() {
let nix_options = hive.nix_options_with_builders().await.unwrap();
let nix_options = hive.nix_flags_with_builders().await.unwrap();
if !info.allows_local_deployment() {
log::error!(
"Local deployment is not enabled for host {}.",
Expand Down
10 changes: 5 additions & 5 deletions src/nix/deployment/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ use futures::future::join_all;
use itertools::Itertools;
use tokio_stream::StreamExt;

use super::NixOptions;
use super::NixFlags;
use crate::job::{JobHandle, JobMonitor, JobState, JobType};
use crate::progress::Sender as ProgressSender;
use crate::util;
Expand Down Expand Up @@ -50,7 +50,7 @@ pub struct Deployment {
options: Options,

/// Options passed to Nix invocations.
nix_options: NixOptions,
nix_options: NixFlags,

/// Handle to send messages to the ProgressOutput.
progress: Option<ProgressSender>,
Expand Down Expand Up @@ -103,7 +103,7 @@ impl Deployment {
hive,
goal,
options: Options::default(),
nix_options: NixOptions::default(),
nix_options: NixFlags::default(),
progress,
targets,
parallelism_limit: ParallelismLimit::default(),
Expand All @@ -129,7 +129,7 @@ impl Deployment {
monitor.set_label_width(width);
}

let nix_options = self.hive.nix_options_with_builders().await?;
let nix_options = self.hive.nix_flags_with_builders().await?;
self.nix_options = nix_options;

if self.goal == Goal::UploadKeys {
Expand Down Expand Up @@ -250,7 +250,7 @@ impl Deployment {
evaluator.set_job(job.clone());

// FIXME: nix-eval-jobs currently does not support IFD with builders
let options = self.hive.nix_options();
let options = self.hive.nix_flags();
let mut stream = evaluator.evaluate(&expr, options).await?;

let mut futures: Vec<tokio::task::JoinHandle<ColmenaResult<()>>> = Vec::new();
Expand Down
4 changes: 2 additions & 2 deletions src/nix/evaluator/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ use std::result::Result as StdResult;
use async_trait::async_trait;
use futures::Stream;

use super::{BuildResult, NixExpression, NixOptions, StoreDerivation, StorePath};
use super::{BuildResult, NixExpression, NixFlags, StoreDerivation, StorePath};
use crate::error::{ColmenaError, ColmenaResult};
use crate::job::JobHandle;

Expand Down Expand Up @@ -61,7 +61,7 @@ pub trait DrvSetEvaluator {
async fn evaluate(
&self,
expression: &dyn NixExpression,
options: NixOptions,
flags: NixFlags,
) -> ColmenaResult<Pin<Box<dyn Stream<Item = EvalResult>>>>;

/// Sets the maximum number of attributes to evaluate at the same time.
Expand Down
14 changes: 7 additions & 7 deletions src/nix/evaluator/nix_eval_jobs.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ use tokio::process::Command;
use super::{AttributeError, AttributeOutput, DrvSetEvaluator, EvalError, EvalResult};
use crate::error::{ColmenaError, ColmenaResult};
use crate::job::{null_job_handle, JobHandle};
use crate::nix::{NixExpression, NixOptions, StorePath};
use crate::nix::{NixExpression, NixFlags, StorePath};
use crate::util::capture_stream;

/// The pinned nix-eval-jobs binary.
Expand Down Expand Up @@ -74,15 +74,15 @@ impl DrvSetEvaluator for NixEvalJobs {
async fn evaluate(
&self,
expression: &dyn NixExpression,
options: NixOptions,
flags: NixFlags,
) -> ColmenaResult<Pin<Box<dyn Stream<Item = EvalResult>>>> {
let mut command = Command::new(&self.executable);
command
.arg("--workers")
.arg(self.workers.to_string())
.args(&["--expr", &expression.expression()]);

command.args(options.to_args());
command.args(flags.to_args());

if expression.requires_flakes() {
command.args(&["--extra-experimental-features", "flakes"]);
Expand Down Expand Up @@ -235,7 +235,7 @@ mod tests {

block_on(async move {
let mut stream = evaluator
.evaluate(&expr, NixOptions::default())
.evaluate(&expr, NixFlags::default())
.await
.unwrap();
let mut count = 0;
Expand All @@ -259,7 +259,7 @@ mod tests {

block_on(async move {
let mut stream = evaluator
.evaluate(&expr, NixOptions::default())
.evaluate(&expr, NixFlags::default())
.await
.unwrap();
let mut count = 0;
Expand All @@ -283,7 +283,7 @@ mod tests {

block_on(async move {
let mut stream = evaluator
.evaluate(&expr, NixOptions::default())
.evaluate(&expr, NixFlags::default())
.await
.unwrap();
let mut count = 0;
Expand Down Expand Up @@ -323,7 +323,7 @@ mod tests {

block_on(async move {
let mut stream = evaluator
.evaluate(&expr, NixOptions::default())
.evaluate(&expr, NixFlags::default())
.await
.unwrap();
let mut count = 0;
Expand Down
47 changes: 28 additions & 19 deletions src/nix/hive/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ use validator::Validate;

use super::deployment::TargetNode;
use super::{
Flake, MetaConfig, NixExpression, NixOptions, NodeConfig, NodeFilter, NodeName,
Flake, MetaConfig, NixExpression, NixFlags, NodeConfig, NodeFilter, NodeName,
ProfileDerivation, SerializedNixExpression, StorePath,
};
use crate::error::ColmenaResult;
Expand Down Expand Up @@ -52,6 +52,9 @@ pub struct Hive {
/// Whether to pass --impure in Nix commands.
impure: bool,

/// Options to pass as --option name value.
nix_options: HashMap<String, String>,

meta_config: OnceCell<MetaConfig>,
}

Expand Down Expand Up @@ -104,6 +107,7 @@ impl Hive {
assets,
show_trace: false,
impure: false,
nix_options: HashMap::new(),
meta_config: OnceCell::new(),
})
}
Expand Down Expand Up @@ -131,25 +135,30 @@ impl Hive {
self.impure = impure;
}

pub fn add_nix_option(&mut self, name: String, value: String) {
self.nix_options.insert(name, value);
}

/// Returns Nix options to set for this Hive.
pub fn nix_options(&self) -> NixOptions {
let mut options = NixOptions::default();
options.set_show_trace(self.show_trace);
options.set_pure_eval(self.path.is_flake());
options.set_impure(self.impure);
options
pub fn nix_flags(&self) -> NixFlags {
let mut flags = NixFlags::default();
flags.set_show_trace(self.show_trace);
flags.set_pure_eval(self.path.is_flake());
flags.set_impure(self.impure);
flags.set_options(self.nix_options.clone());
flags
}

/// Returns Nix options to set for this Hive, with configured remote builders.
pub async fn nix_options_with_builders(&self) -> ColmenaResult<NixOptions> {
let mut options = NixOptions::default();
options.set_show_trace(self.show_trace);
/// Returns Nix flags to set for this Hive, with configured remote builders.
pub async fn nix_flags_with_builders(&self) -> ColmenaResult<NixFlags> {
let mut flags = NixFlags::default();
flags.set_show_trace(self.show_trace);

if let Some(machines_file) = &self.get_meta_config().await?.machines_file {
options.set_builders(Some(format!("@{}", machines_file)));
flags.set_builders(Some(format!("@{}", machines_file)));
}

Ok(options)
Ok(flags)
}

/// Convenience wrapper to filter nodes for CLI actions.
Expand Down Expand Up @@ -423,32 +432,32 @@ impl<'hive> NixInstantiate<'hive> {

fn eval(self) -> Command {
let mut command = self.instantiate();
let options = self.hive.nix_options();
let flags = self.hive.nix_flags();
command
.arg("--eval")
.arg("--json")
.arg("--strict")
// Ensures the derivations are instantiated
// Required for system profile evaluation and IFD
.arg("--read-write-mode")
.args(options.to_args());
.args(flags.to_args());
command
}

async fn instantiate_with_builders(self) -> ColmenaResult<Command> {
let options = self.hive.nix_options_with_builders().await?;
let flags = self.hive.nix_flags_with_builders().await?;
let mut command = self.instantiate();

command.args(options.to_args());
command.args(flags.to_args());

Ok(command)
}

async fn eval_with_builders(self) -> ColmenaResult<Command> {
let options = self.hive.nix_options_with_builders().await?;
let flags = self.hive.nix_flags_with_builders().await?;
let mut command = self.eval();

command.args(options.to_args());
command.args(flags.to_args());

Ok(command)
}
Expand Down
6 changes: 3 additions & 3 deletions src/nix/host/local.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ use tokio::process::Command;
use super::{key_uploader, CopyDirection, CopyOptions, Host};
use crate::error::{ColmenaError, ColmenaResult};
use crate::job::JobHandle;
use crate::nix::{Goal, Key, NixOptions, Profile, StorePath, CURRENT_PROFILE, SYSTEM_PROFILE};
use crate::nix::{Goal, Key, NixFlags, Profile, StorePath, CURRENT_PROFILE, SYSTEM_PROFILE};
use crate::util::{CommandExecution, CommandExt};

/// The local machine running Colmena.
Expand All @@ -18,12 +18,12 @@ use crate::util::{CommandExecution, CommandExt};
#[derive(Debug)]
pub struct Local {
job: Option<JobHandle>,
nix_options: NixOptions,
nix_options: NixFlags,
privilege_escalation_command: Option<Vec<String>>,
}

impl Local {
pub fn new(nix_options: NixOptions) -> Self {
pub fn new(nix_options: NixFlags) -> Self {
Self {
job: None,
nix_options,
Expand Down
31 changes: 22 additions & 9 deletions src/nix/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,9 +91,9 @@ pub struct MetaConfig {
pub machines_file: Option<String>,
}

/// Nix options.
/// Nix CLI flags.
#[derive(Debug, Clone, Default)]
pub struct NixOptions {
pub struct NixFlags {
/// Whether to pass --show-trace.
show_trace: bool,

Expand All @@ -111,6 +111,9 @@ pub struct NixOptions {
/// - `@/path/to/machines`
/// - `builder@host.tld riscv64-linux /home/nix/.ssh/keys/builder.key 8 1 kvm`
builders: Option<String>,

/// Options to pass as --option name value.
options: HashMap<String, String>,
}

impl NodeName {
Expand Down Expand Up @@ -188,7 +191,7 @@ impl NodeConfig {
}
}

impl NixOptions {
impl NixFlags {
pub fn set_show_trace(&mut self, show_trace: bool) {
self.show_trace = show_trace;
}
Expand All @@ -205,30 +208,40 @@ impl NixOptions {
self.builders = builders;
}

pub fn set_options(&mut self, options: HashMap<String, String>) {
self.options = options;
}

pub fn to_args(&self) -> Vec<String> {
let mut options = Vec::new();
let mut args = Vec::new();

if let Some(builders) = &self.builders {
options.append(&mut vec![
args.append(&mut vec![
"--option".to_string(),
"builders".to_string(),
builders.clone(),
]);
}

if self.show_trace {
options.push("--show-trace".to_string());
args.push("--show-trace".to_string());
}

if self.pure_eval {
options.push("--pure-eval".to_string());
args.push("--pure-eval".to_string());
}

if self.impure {
options.push("--impure".to_string());
args.push("--impure".to_string());
}

for (name, value) in self.options.iter() {
args.push("--option".to_string());
args.push(name.to_string());
args.push(value.to_string());
}

options
args
}
}

Expand Down
Loading