new: Support env mode scoped lockfiles. (#1082)

Author: milesj

CHANGELOG.md

@@ -17,14 +17,31 @@
 
 ## Unreleased
 
+#### 💥 Breaking
+
+- **Lockfiles**
+  - Added support for environment scoped lockfiles. When `PROTO_ENV` is set, a `.prototools.<env>` config is now locked to a sibling `.protolock.<env>` lockfile, instead of sharing the directory's `.protolock` with the base `.prototools` config.
+    - Each lockfile only tracks tools with versions defined in its own config. A version overridden by an environment config is tracked in `.protolock.<env>`, while versions inherited from `.prototools` remain in `.protolock`. Ad-hoc installs are tracked by the base `.prototools` config.
+    - Environment configs inherit the `settings.lockfile` setting from the `.prototools` config in the same directory, but can override it. Lockfiles for inactive environments are never loaded or modified.
+    - Records for environment specific versions that were previously written to `.protolock` are no longer used, and will be re-created in `.protolock.<env>` on the next install.
+
 #### 🚀 Updates
 
+- **Lockfiles**
+  - Updated `proto pin` and `proto unpin` to always keep the lockfile of the modified config in sync, even when another config (like an environment config) takes precedence for the tool.
+  - Updated `proto install --pin` to track the record of the installed version in the lockfile of the pinned config, even when another config (like an environment config) takes precedence for the tool.
+  - Updated `proto uninstall` to remove records for the uninstalled version from all applicable lockfiles, as the version may be pinned in multiple configs (each of which is unpinned). When uninstalling all versions, the tool is removed from all applicable lockfiles.
+  - Updated `proto outdated --update` to also update versions defined in environment scoped configs (`.prototools.<env>`), including the records in their lockfiles. Previously these versions were skipped with a warning.
 - **WASM API**
   - Added a `download_file` host function, which downloads a file from a URL directly to a file on the host machine, without loading the contents into WASM memory. Requests are made with proto's HTTP client, respecting `[settings.http]` and `.netrc` configuration.
     - Added `download` and `download_from_url` functions, and a `download_file!` macro, to the PDK.
   - Added a `method` field to the `send_request` host function input (`SendRequestInput`), which supports `GET` (default) and `POST`.
   - Added a `SendRequestInput::post()` constructor for creating `POST` requests.
 
+#### 🐞 Fixes
+
+- Fixed an issue where `proto uninstall` would fail when `PROTO_ENV` is set and an environment scoped `.prototools.<env>` config exists, as it would attempt to unpin the version from an invalid path.
+
 ## 0.60.2
 
 #### 🚀 Updates

crates/cli/src/commands/debug/config.rs

@@ -39,9 +39,9 @@ pub async fn config(session: ProtoSession, args: DebugConfigArgs) -> SessionResu
     if session.is_json_format() {
         let mut locks = BTreeMap::default();
 
-        for entry in &manager.entries {
-            if entry.locked
-                && let Some(lock) = manager.get_lock(&entry.path)?
+        for file in manager.get_config_files() {
+            if file.locked
+                && let Some(lock) = manager.get_lock(&file.path)?
             {
                 locks.insert(lock.path.clone(), (*lock).clone());
             }
@@ -61,25 +61,25 @@ pub async fn config(session: ProtoSession, args: DebugConfigArgs) -> SessionResu
         return Ok(None);
     }
 
-    for entry in manager.entries.iter().rev() {
-        for file in &entry.configs {
-            if file.exists {
-                let code = toml::format(&file.config, true)?;
-
-                session.console.render(element! {
-                    Container {
-                        Section(
-                            title: file.path.to_string_lossy(),
-                            title_color: style_to_color(Style::Path)
-                        )
-                        CodeBlock(code, format: "toml")
-                    }
-                })?;
-            }
+    // Render from lowest to highest precedence, with each
+    // config followed by the lockfile it enabled
+    for file in manager.get_config_files().into_iter().rev() {
+        if file.exists {
+            let code = toml::format(&file.config, true)?;
+
+            session.console.render(element! {
+                Container {
+                    Section(
+                        title: file.path.to_string_lossy(),
+                        title_color: style_to_color(Style::Path)
+                    )
+                    CodeBlock(code, format: "toml")
+                }
+            })?;
         }
 
-        if entry.locked
-            && let Some(lock) = manager.get_lock(&entry.path)?
+        if file.locked
+            && let Some(lock) = manager.get_lock(&file.path)?
         {
             let code = toml::format(&*lock, true)?;
 

crates/cli/src/commands/diagnose.rs

@@ -298,25 +298,23 @@ fn gather_lockfile_warnings(session: &ProtoSession) -> Result<Vec<Issue>, ProtoC
     let mut warnings = vec![];
     let manager = session.env.load_file_manager()?;
 
-    for entry in &manager.entries {
-        if !entry.locked {
+    for file in manager.get_config_files() {
+        if !file.locked {
             continue;
         }
 
-        let Some(lock) = manager.get_lock(&entry.path)? else {
+        let Some(lock) = manager.get_lock(&file.path)? else {
             continue;
         };
 
-        // Gather specs defined in sibling configs, so that we can
-        // detect lockfile records that no longer match a config
+        // Gather specs defined in the config that owns the lockfile, so
+        // that we can detect lockfile records that no longer match it
         let mut config_specs: BTreeMap<&ToolContext, BTreeSet<&UnresolvedVersionSpec>> =
             BTreeMap::default();
 
-        for file in &entry.configs {
-            if let Some(versions) = &file.config.versions {
-                for (context, spec) in versions {
-                    config_specs.entry(context).or_default().insert(&spec.req);
-                }
+        if let Some(versions) = &file.config.versions {
+            for (context, spec) in versions {
+                config_specs.entry(context).or_default().insert(&spec.req);
             }
         }
 

crates/cli/src/commands/outdated.rs

@@ -250,7 +250,9 @@ pub async fn outdated(session: ProtoSession, args: OutdatedArgs) -> SessionResul
                 continue;
             };
 
-            if !src.ends_with(PROTO_CONFIG_NAME) {
+            // Only proto configs can be updated, including environment scoped
+            // configs, as versions may also be detected from ecosystem files
+            if !ProtoConfig::is_config_file(src) {
                 warn!(
                     config = ?src,
                     "Unable to update the version for {}, as its config source is not a {} file",
@@ -297,23 +299,25 @@ pub async fn outdated(session: ProtoSession, args: OutdatedArgs) -> SessionResul
             })?;
         }
 
-        // Update lockfile records to match the newly updated configs,
+        // Update records in the lockfiles owned by the updated configs,
         // otherwise the stale records will be used indefinitely
         for tool in &tools {
             let Some(item) = items.get(&tool.context) else {
                 continue;
             };
 
-            let Some(new_spec) = item
-                .config_source
-                .as_ref()
-                .and_then(|src| updates.get(src))
+            let Some(src) = &item.config_source else {
+                continue;
+            };
+
+            let Some(new_spec) = updates
+                .get(src)
                 .and_then(|versions| versions.get(&tool.context))
             else {
                 continue;
             };
 
-            Locker::new(tool).update_spec_in_lockfile(
+            Locker::for_config(tool, src).update_spec_in_lockfile(
                 &item.config_version.req,
                 new_spec,
                 if args.latest {

crates/cli/src/commands/pin.rs

@@ -3,7 +3,7 @@ use clap::Args;
 use proto_core::flow::lock::{Locker, ProtoLockError};
 use proto_core::flow::resolve::Resolver;
 use proto_core::{
-    PinLocation, ProtoConfig, Tool, ToolContext, ToolSpec, cfg, reporter::NoticeOutput,
+    LockRecord, PinLocation, ProtoConfig, Tool, ToolContext, ToolSpec, cfg, reporter::NoticeOutput,
 };
 use proto_pdk_api::{PinVersionInput, PinVersionOutput, PluginFunction};
 use starbase_console::ui::*;
@@ -29,10 +29,15 @@ pub struct PinArgs {
     pub tool_native: bool,
 }
 
+/// Pin the version to the config at the provided location, and keep the
+/// lockfile owned by that config in sync. When pinning as part of an install,
+/// the record of the fresh install should be provided, so that it's tracked
+/// by the pinned config, which now defines a version for the tool.
 pub async fn internal_pin(
     tool: &Tool,
     spec: &ToolSpec,
     pin_to: PinLocation,
+    install_record: Option<&LockRecord>,
 ) -> Result<PathBuf, ProtoLockError> {
     let version = match &spec.version {
         Some(version) => version.to_string(),
@@ -58,42 +63,40 @@ pub async fn internal_pin(
         "Pinned the version",
     );
 
-    // Keep lockfile records in sync with the config change, but only when
-    // the config being modified owns the lock records for the tool
-    let owns_lock = tool
-        .proto
-        .load_file_manager()?
-        .get_locked_dir(&tool.context)
-        .is_some_and(|dir| dir == config_dir);
-
-    if owns_lock {
-        let locker = Locker::new(tool);
-
-        match &spec.version {
-            // We know what the pinned version resolves to, so migrate
-            // records from the requested and previous specs to it
-            Some(new_version) => {
-                let new_spec = new_version.to_unresolved_spec();
-
-                locker.update_spec_in_lockfile(&spec.req, &new_spec, new_version)?;
-
-                if let Some(previous) = previous_spec
-                    && previous.req != spec.req
-                {
-                    locker.update_spec_in_lockfile(&previous.req, &new_spec, new_version)?;
-                }
+    // Keep records in the lockfile owned by the modified config in sync
+    // with the change (no-op if the config has not enabled a lockfile)
+    let locker = Locker::for_config(tool, &config_path);
+
+    // The install record was tracked by the config that previously defined
+    // the tool (if any), which may not be the config we just pinned to
+    if let Some(record) = install_record {
+        locker.insert_record_into_lockfile(record)?;
+    }
+
+    match &spec.version {
+        // We know what the pinned version resolves to, so migrate
+        // records from the requested and previous specs to it
+        Some(new_version) => {
+            let new_spec = new_version.to_unresolved_spec();
+
+            locker.update_spec_in_lockfile(&spec.req, &new_spec, new_version)?;
+
+            if let Some(previous) = previous_spec
+                && previous.req != spec.req
+            {
+                locker.update_spec_in_lockfile(&previous.req, &new_spec, new_version)?;
             }
-            // We don't know what the pinned version resolves to, so
-            // remove records for the previous spec
-            None => {
-                if let Some(previous) = previous_spec
-                    && previous.req != spec.req
-                {
-                    locker.remove_spec_from_lockfile(&previous.req)?;
-                }
+        }
+        // We don't know what the pinned version resolves to, so
+        // remove records for the previous spec
+        None => {
+            if let Some(previous) = previous_spec
+                && previous.req != spec.req
+            {
+                locker.remove_spec_from_lockfile(&previous.req)?;
             }
-        };
-    }
+        }
+    };
 
     Ok(config_path)
 }
@@ -158,7 +161,7 @@ pub async fn pin(session: ProtoSession, args: PinArgs) -> SessionResult {
             return Ok(Some(1));
         }
     } else {
-        config_path = internal_pin(&tool, &spec, args.to).await?;
+        config_path = internal_pin(&tool, &spec, args.to, None).await?;
     }
 
     session.console.notice(

crates/cli/src/commands/uninstall.rs

@@ -87,8 +87,12 @@ async fn try_uninstall_all(tool: &mut ToolRecord) -> miette::Result<()> {
     fs::remove_dir_all(tool.get_inventory_dir())?;
     fs::remove_dir_all(tool.get_temp_dir())?;
 
-    // Remove from lockfile
-    Locker::new(tool).remove_from_lockfile()?;
+    // Remove from all lockfiles, as the tool is unpinned from all configs
+    for file in tool.proto.load_file_manager()?.get_config_files() {
+        if file.locked {
+            Locker::for_config(tool, &file.path).remove_from_lockfile()?;
+        }
+    }
 
     Ok(())
 }

crates/cli/src/commands/unpin.rs

@@ -90,16 +90,10 @@ pub async fn unpin(session: ProtoSession, args: UnpinArgs) -> SessionResult {
                 });
         })?;
 
-        // Remove lockfile records for the unpinned spec, but only when
-        // the config being modified owns the lock records for the tool
-        if let Some(removed) = removed_spec
-            && tool
-                .proto
-                .load_file_manager()?
-                .get_locked_dir(&tool.context)
-                .is_some_and(|dir| dir == config_dir)
-        {
-            Locker::new(&tool).remove_spec_from_lockfile(&removed.req)?;
+        // Remove records for the unpinned spec from the lockfile owned by
+        // the modified config (no-op if the config has not enabled a lockfile)
+        if let Some(removed) = removed_spec {
+            Locker::for_config(&tool, &config_path).remove_spec_from_lockfile(&removed.req)?;
         }
     }
 

crates/cli/src/workflows/install_workflow.rs

@@ -112,7 +112,7 @@ impl InstallWorkflow {
         Resolver::resolve(&self.tool, spec, false).await?;
 
         if !params.force && self.tool.is_installed(spec) {
-            self.pin_version(spec, &params.pin_to).await?;
+            self.pin_version(spec, None, params.pin_to.as_ref()).await?;
             self.finish_progress(spec, started);
 
             // Ensure bins/shims exist
@@ -125,13 +125,21 @@ impl InstallWorkflow {
         self.pre_install(spec, &params).await?;
 
         // Run install
-        let record = self.do_install(spec, &params).await?;
-
-        if record.is_none() {
+        let Some(record) = self.do_install(spec, &params).await? else {
             return Ok(InstallOutcome::FailedToInstall(self.tool.get_id().clone()));
-        }
+        };
 
-        let pinned = self.pin_version(spec, &params.pin_to).await?;
+        let pinned = self
+            .pin_version(
+                spec,
+                if spec.update_lockfile {
+                    Some(&record)
+                } else {
+                    None
+                },
+                params.pin_to.as_ref(),
+            )
+            .await?;
         self.finish_progress(spec, started);
 
         // Run post-install hooks
@@ -396,7 +404,8 @@ impl InstallWorkflow {
     async fn pin_version(
         &mut self,
         spec: &ToolSpec,
-        arg_pin_to: &Option<PinLocation>,
+        install_record: Option<&LockRecord>,
+        arg_pin_to: Option<&PinLocation>,
     ) -> Result<bool, ProtoCliError> {
         let config = self.tool.proto.load_config()?;
         let mut pin_to = PinLocation::Local;
@@ -422,7 +431,7 @@ impl InstallWorkflow {
         }
 
         if pin {
-            internal_pin(&self.tool.tool, spec, pin_to).await?;
+            internal_pin(&self.tool.tool, spec, pin_to, install_record).await?;
         }
 
         Ok(pin)