Skip to content

Add option to build on the target host #175

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Nov 18, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -183,6 +183,11 @@ This is a set of options that can be put in any of the above definitions, with t
# If not specified, this will default to `/tmp`
# (if `magicRollback` is in use, this _must_ be writable by `user`)
tempPath = "/home/someuser/.deploy-rs";

# Build the derivation on the target system.
# Will also fetch all external dependencies from the target system's substituters.
# This default to `false`
remoteBuild = true;
}
```

Expand Down
13 changes: 7 additions & 6 deletions src/cli.rs
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,10 @@ pub struct Opts {
#[clap(short, long)]
skip_checks: bool,

/// Build on remote host
#[clap(long)]
remote_build: bool,

/// Override the SSH user with the given value
#[clap(long)]
ssh_user: Option<String>,
Expand Down Expand Up @@ -138,9 +142,7 @@ async fn check_deployment(
.arg(format!("let r = import {}/.; x = (if builtins.isFunction r then (r {{}}) else r); in if x ? checks then x.checks.${{builtins.currentSystem}} else {{}}", repo));
}

for extra_arg in extra_build_args {
check_command.arg(extra_arg);
}
check_command.args(extra_build_args);

let check_status = check_command.status().await?;

Expand Down Expand Up @@ -239,9 +241,7 @@ async fn get_deployment_data(
.arg(format!("let r = import {}/.; in if builtins.isFunction r then (r {{}}).deploy else r.deploy", flake.repo))
};

for extra_arg in extra_build_args {
c.arg(extra_arg);
}
c.args(extra_build_args);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you also add --remote-build flag to the command line arguments of deploy?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

as a "build all deployments on the respective target servers" override?
the current implementation only builds selected builds remotely, but that could be a good idea

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

as a "build all deployments on the respective target servers" override?

Yes, I imagine that it might be useful when one doesn't have decent internet connection or has limited traffic


let build_child = c
.stdout(Stdio::piped())
Expand Down Expand Up @@ -640,6 +640,7 @@ pub async fn run(args: Option<&ArgMatches>) -> Result<(), RunError> {
temp_path: opts.temp_path,
confirm_timeout: opts.confirm_timeout,
dry_activate: opts.dry_activate,
remote_build: opts.remote_build,
sudo: opts.sudo,
};

Expand Down
2 changes: 2 additions & 0 deletions src/data.rs
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@ pub struct GenericSettings {
pub magic_rollback: Option<bool>,
#[serde(rename(deserialize = "sudo"))]
pub sudo: Option<String>,
#[serde(default,rename(deserialize = "remoteBuild"))]
pub remote_build: Option<bool>,
}

#[derive(Deserialize, Debug, Clone)]
Expand Down
13 changes: 9 additions & 4 deletions src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,7 @@ pub struct CmdOverrides {
pub confirm_timeout: Option<u16>,
pub sudo: Option<String>,
pub dry_activate: bool,
pub remote_build: bool,
}

#[derive(PartialEq, Debug)]
Expand Down Expand Up @@ -395,10 +396,10 @@ impl<'a> DeployData<'a> {
}

fn get_sudo(&'a self) -> String {
return match self.merged_settings.sudo {
Some(ref x) => x.clone(),
None => "sudo -u".to_string()
};
match self.merged_settings.sudo {
Some(ref x) => x.clone(),
None => "sudo -u".to_string(),
}
}
}

Expand All @@ -416,6 +417,10 @@ pub fn make_deploy_data<'a, 's>(
merged_settings.merge(node.generic_settings.clone());
merged_settings.merge(top_settings.clone());

// build all machines remotely when the command line flag is set
if cmd_overrides.remote_build {
merged_settings.remote_build = Some(cmd_overrides.remote_build);
}
if cmd_overrides.ssh_user.is_some() {
merged_settings.ssh_user = cmd_overrides.ssh_user.clone();
}
Expand Down
194 changes: 135 additions & 59 deletions src/push.rs
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ pub enum PushProfileError {
Copy(std::io::Error),
#[error("Nix copy command resulted in a bad exit code: {0:?}")]
CopyExit(Option<i32>),
#[error("The remote building option is not supported when using legacy nix")]
RemoteBuildWithLegacyNix,
}

pub struct PushProfileData<'a> {
Expand All @@ -54,40 +56,7 @@ pub struct PushProfileData<'a> {
pub extra_build_args: &'a [String],
}

pub async fn push_profile(data: PushProfileData<'_>) -> Result<(), PushProfileError> {
debug!(
"Finding the deriver of store path for {}",
&data.deploy_data.profile.profile_settings.path
);

// `nix-store --query --deriver` doesn't work on invalid paths, so we parse output of show-derivation :(
let mut show_derivation_command = Command::new("nix");

show_derivation_command
.arg("show-derivation")
.arg(&data.deploy_data.profile.profile_settings.path);

let show_derivation_output = show_derivation_command
.output()
.await
.map_err(PushProfileError::ShowDerivation)?;

match show_derivation_output.status.code() {
Some(0) => (),
a => return Err(PushProfileError::ShowDerivationExit(a)),
};

let derivation_info: HashMap<&str, serde_json::value::Value> = serde_json::from_str(
std::str::from_utf8(&show_derivation_output.stdout)
.map_err(PushProfileError::ShowDerivationUtf8)?,
)
.map_err(PushProfileError::ShowDerivationParse)?;

let derivation_name = derivation_info
.keys()
.next()
.ok_or(PushProfileError::ShowDerivationEmpty)?;

pub async fn build_profile_locally(data: &PushProfileData<'_>, derivation_name: &str) -> Result<(), PushProfileError> {
info!(
"Building profile `{}` for node `{}`",
data.deploy_data.profile_name, data.deploy_data.node_name
Expand Down Expand Up @@ -118,9 +87,7 @@ pub async fn push_profile(data: PushProfileData<'_>) -> Result<(), PushProfileEr
(false, true) => build_command.arg("--no-link"),
};

for extra_arg in data.extra_build_args {
build_command.arg(extra_arg);
}
build_command.args(data.extra_build_args);

let build_exit_status = build_command
// Logging should be in stderr, this just stops the store path from printing for no reason

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

When I try this out and run it on my machine, it fails on line 181

  if !Path::new(
        format!(
            "{}/deploy-rs-activate",
            data.deploy_data.profile.profile_settings.path
        )
        .as_str(),

because, as the system is remote and the closure was built remotely, this path is not going to exist in my store or on my system.

I'm not sure I have a great solution for that, but I wanted to point that out cause I'm trying this out in anticipation of managing remote machines via macOS

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Perhaps, in this case, activation should happen on the target as well 🤔

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

FTR, I have the same issue even with NixOS -> NixOS deployment 🙈

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How I ended up "sorta solving this" myself was to take the path that deploy info spits out and then dead-reckoning run a little bash function with that path as the input.

function activate-on-host(){                                                            
  printf -v profile %q "$1"                                                          
  ssh user@FQDN "export PROFILE=$profile ; bash $profile/deploy-rs-activate" 
}                                                                                    

Which, indeed, simply runs the activation on the target

Copy link
Collaborator Author

@PhilTaken PhilTaken Nov 16, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see, I did not encounter that error, very likely because I had it built locally before I tested remote building

the push_profile function does quite a lote at this point and most of it assumes the derivation is being built locally

splitting it up into a function to handle building locally and another to handle remote builds should resolve some of those difficulties

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

besides, contrary to its name it also handles the building, not just the pushing 🤔

Expand Down Expand Up @@ -179,22 +146,77 @@ pub async fn push_profile(data: PushProfileData<'_>) -> Result<(), PushProfileEr
a => return Err(PushProfileError::SignExit(a)),
};
}
Ok(())
}

pub async fn build_profile_remotely(data: &PushProfileData<'_>, derivation_name: &str) -> Result<(), PushProfileError> {
info!(
"Copying profile `{}` to node `{}`",
"Building profile `{}` for node `{}` on remote host",
data.deploy_data.profile_name, data.deploy_data.node_name
);

let mut copy_command = Command::new("nix");
copy_command.arg("copy");
let store_address = format!("ssh-ng://{}@{}",
if data.deploy_data.profile.generic_settings.ssh_user.is_some() {
&data.deploy_data.profile.generic_settings.ssh_user.as_ref().unwrap()
} else {
&data.deploy_defs.ssh_user
},
data.deploy_data.node.node_settings.hostname
);

if data.deploy_data.merged_settings.fast_connection != Some(true) {
copy_command.arg("--substitute-on-destination");
}
let ssh_opts_str = data.deploy_data.merged_settings.ssh_opts.join(" ");

if !data.check_sigs {
copy_command.arg("--no-check-sigs");
}

// copy the derivation to remote host so it can be built there
let copy_command_status = Command::new("nix").arg("copy")
.arg("-s") // fetch dependencies from substitures, not localhost
.arg("--to").arg(&store_address)
.arg("--derivation").arg(derivation_name)
.env("NIX_SSHOPTS", ssh_opts_str.clone())
.stdout(Stdio::null())
.status()
.await
.map_err(PushProfileError::Copy)?;

match copy_command_status.code() {
Some(0) => (),
a => return Err(PushProfileError::CopyExit(a)),
};

let mut build_command = Command::new("nix");
build_command
.arg("build").arg(derivation_name)
.arg("--eval-store").arg("auto")
.arg("--store").arg(&store_address)
.args(data.extra_build_args)
.env("NIX_SSHOPTS", ssh_opts_str.clone());

debug!("build command: {:?}", build_command);

let build_exit_status = build_command
// Logging should be in stderr, this just stops the store path from printing for no reason
.stdout(Stdio::null())
.status()
.await
.map_err(PushProfileError::Build)?;

match build_exit_status.code() {
Some(0) => (),
a => return Err(PushProfileError::BuildExit(a)),
};


Ok(())
}

pub async fn push_profile(data: PushProfileData<'_>) -> Result<(), PushProfileError> {
debug!(
"Finding the deriver of store path for {}",
&data.deploy_data.profile.profile_settings.path
);

// `nix-store --query --deriver` doesn't work on invalid paths, so we parse output of show-derivation :(
let mut show_derivation_command = Command::new("nix");

let ssh_opts_str = data
.deploy_data
Expand All @@ -206,24 +228,78 @@ pub async fn push_profile(data: PushProfileData<'_>) -> Result<(), PushProfileEr
// .collect::<Vec<String>>()
.join(" ");

let hostname = match data.deploy_data.cmd_overrides.hostname {
Some(ref x) => x,
None => &data.deploy_data.node.node_settings.hostname,
};

let copy_exit_status = copy_command
.arg("--to")
.arg(format!("ssh://{}@{}", data.deploy_defs.ssh_user, hostname))
.arg(&data.deploy_data.profile.profile_settings.path)
.env("NIX_SSHOPTS", ssh_opts_str)
.status()
show_derivation_command
.arg("show-derivation")
.arg(&data.deploy_data.profile.profile_settings.path);

let show_derivation_output = show_derivation_command
.output()
.await
.map_err(PushProfileError::Copy)?;
.map_err(PushProfileError::ShowDerivation)?;

match copy_exit_status.code() {
match show_derivation_output.status.code() {
Some(0) => (),
a => return Err(PushProfileError::CopyExit(a)),
a => return Err(PushProfileError::ShowDerivationExit(a)),
};

let derivation_info: HashMap<&str, serde_json::value::Value> = serde_json::from_str(
std::str::from_utf8(&show_derivation_output.stdout)
.map_err(PushProfileError::ShowDerivationUtf8)?,
)
.map_err(PushProfileError::ShowDerivationParse)?;

let derivation_name = derivation_info
.keys()
.next()
.ok_or(PushProfileError::ShowDerivationEmpty)?;

if data.deploy_data.merged_settings.remote_build.unwrap_or(false) {
if !data.supports_flakes {
return Err(PushProfileError::RemoteBuildWithLegacyNix)
}

// remote building guarantees that the resulting derivation is stored on the target system
// no need to copy after building
build_profile_remotely(&data, derivation_name).await?;
} else {
build_profile_locally(&data, derivation_name).await?;

info!(
"Copying profile `{}` to node `{}`",
data.deploy_data.profile_name, data.deploy_data.node_name
);

let mut copy_command = Command::new("nix");
copy_command.arg("copy");

if data.deploy_data.merged_settings.fast_connection != Some(true) {
copy_command.arg("--substitute-on-destination");
}

if !data.check_sigs {
copy_command.arg("--no-check-sigs");
}

let hostname = match data.deploy_data.cmd_overrides.hostname {
Some(ref x) => x,
None => &data.deploy_data.node.node_settings.hostname,
};

let copy_exit_status = copy_command
.arg("--to")
.arg(format!("ssh://{}@{}", data.deploy_defs.ssh_user, hostname))
.arg(&data.deploy_data.profile.profile_settings.path)
.env("NIX_SSHOPTS", ssh_opts_str)
.status()
.await
.map_err(PushProfileError::Copy)?;

match copy_exit_status.code() {
Some(0) => (),
a => return Err(PushProfileError::CopyExit(a)),
};
}

Ok(())
}