From 07ee45e3450ed5ba18077987ea09c28f09b70054 Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Tue, 16 Jul 2024 11:15:54 +1000 Subject: [PATCH 1/6] Cache python interpreter info in file --- crates/pet-linux-global-python/src/lib.rs | 1 + crates/pet-mac-commandlinetools/src/lib.rs | 28 ++- crates/pet-mac-xcode/src/lib.rs | 27 ++- crates/pet-python-utils/src/cache.rs | 228 +++++++++++++-------- crates/pet-python-utils/src/env.rs | 41 +++- crates/pet-python-utils/src/fs_cache.rs | 122 +++++++++++ crates/pet-python-utils/src/lib.rs | 1 + crates/pet/src/lib.rs | 13 +- crates/pet/src/main.rs | 14 +- crates/pet/src/resolve.rs | 8 +- 10 files changed, 368 insertions(+), 115 deletions(-) create mode 100644 crates/pet-python-utils/src/fs_cache.rs diff --git a/crates/pet-linux-global-python/src/lib.rs b/crates/pet-linux-global-python/src/lib.rs index b42a691e..e62f1bf1 100644 --- a/crates/pet-linux-global-python/src/lib.rs +++ b/crates/pet-linux-global-python/src/lib.rs @@ -115,6 +115,7 @@ fn find_and_report_global_pythons_in( } if let Some(resolved) = ResolvedPythonEnv::from(exe) { if let Some(env) = get_python_in_bin(&resolved.to_python_env(), resolved.is64_bit) { + resolved.add_to_cache(env.clone()); let mut reported_executables = reported_executables.lock().unwrap(); // env.symlinks = Some([symlinks, env.symlinks.clone().unwrap_or_default()].concat()); if let Some(symlinks) = &env.symlinks { diff --git a/crates/pet-mac-commandlinetools/src/lib.rs b/crates/pet-mac-commandlinetools/src/lib.rs index 0ad3aa63..382dc6c1 100644 --- a/crates/pet-mac-commandlinetools/src/lib.rs +++ b/crates/pet-mac-commandlinetools/src/lib.rs @@ -97,6 +97,8 @@ impl Locator for MacCmdLineTools { } } + let mut resolved_environments = vec![]; + // We know /usr/bin/python3 can end up pointing to this same Python exe as well // Hence look for those symlinks as well. // Unfortunately /usr/bin/python3 is not a real symlink @@ -105,6 +107,8 @@ impl Locator for MacCmdLineTools { if !symlinks.contains(&possible_exes) { if let Some(resolved_env) = ResolvedPythonEnv::from(&possible_exes) { if symlinks.contains(&resolved_env.executable) { + resolved_environments.push(resolved_env.clone()); + symlinks.push(possible_exes); // Use the latest accurate information we have. version = Some(resolved_env.version); @@ -163,19 +167,27 @@ impl Locator for MacCmdLineTools { } if version.is_none() || prefix.is_none() { if let Some(resolved_env) = ResolvedPythonEnv::from(&env.executable) { + resolved_environments.push(resolved_env.clone()); version = Some(resolved_env.version); prefix = Some(resolved_env.prefix); } } - Some( - PythonEnvironmentBuilder::new(Some(PythonEnvironmentKind::MacCommandLineTools)) - .executable(Some(env.executable.clone())) - .version(version) - .prefix(prefix) - .symlinks(Some(symlinks)) - .build(), - ) + let env = PythonEnvironmentBuilder::new(Some(PythonEnvironmentKind::MacCommandLineTools)) + .executable(Some(env.executable.clone())) + .version(version) + .prefix(prefix) + .symlinks(Some(symlinks.clone())) + .build(); + + // If we had spawned Python, then ensure we cache the details. + // We do this here, to ensure we keep track of the symlinks as well, + // I.e. if any of the symlinks change, then the cache is invalidated. + for resolved_env in resolved_environments { + resolved_env.add_to_cache(env.clone()); + } + + Some(env) } fn find(&self, _reporter: &dyn Reporter) { diff --git a/crates/pet-mac-xcode/src/lib.rs b/crates/pet-mac-xcode/src/lib.rs index a6575dff..05f36090 100644 --- a/crates/pet-mac-xcode/src/lib.rs +++ b/crates/pet-mac-xcode/src/lib.rs @@ -95,6 +95,8 @@ impl Locator for MacXCode { } } + let mut resolved_environments = vec![]; + // We know /usr/bin/python3 can end up pointing to this same Python exe as well // Hence look for those symlinks as well. // Unfortunately /usr/bin/python3 is not a real symlink @@ -103,6 +105,7 @@ impl Locator for MacXCode { if !symlinks.contains(&possible_exes) { if let Some(resolved_env) = ResolvedPythonEnv::from(&possible_exes) { if symlinks.contains(&resolved_env.executable) { + resolved_environments.push(resolved_env.clone()); symlinks.push(possible_exes); // Use the latest accurate information we have. version = Some(resolved_env.version); @@ -150,19 +153,27 @@ impl Locator for MacXCode { } if version.is_none() || prefix.is_none() { if let Some(resolved_env) = ResolvedPythonEnv::from(&env.executable) { + resolved_environments.push(resolved_env.clone()); version = Some(resolved_env.version); prefix = Some(resolved_env.prefix); } } - Some( - PythonEnvironmentBuilder::new(Some(PythonEnvironmentKind::MacXCode)) - .executable(Some(env.executable.clone())) - .version(version) - .prefix(prefix) - .symlinks(Some(symlinks)) - .build(), - ) + let env = PythonEnvironmentBuilder::new(Some(PythonEnvironmentKind::MacXCode)) + .executable(Some(env.executable.clone())) + .version(version) + .prefix(prefix) + .symlinks(Some(symlinks)) + .build(); + + // If we had spawned Python, then ensure we cache the details. + // We do this here, to ensure we keep track of the symlinks as well, + // I.e. if any of the symlinks change, then the cache is invalidated. + for resolved_env in resolved_environments { + resolved_env.add_to_cache(env.clone()); + } + + Some(env) } fn find(&self, _reporter: &dyn Reporter) { diff --git a/crates/pet-python-utils/src/cache.rs b/crates/pet-python-utils/src/cache.rs index d4edafee..cbaf446c 100644 --- a/crates/pet-python-utils/src/cache.rs +++ b/crates/pet-python-utils/src/cache.rs @@ -2,24 +2,27 @@ // Licensed under the MIT License. use lazy_static::lazy_static; -use pet_fs::path::norm_case; -use sha2::{Digest, Sha256}; +use log::trace; use std::{ - collections::{hash_map::Entry, HashMap}, + collections::{hash_map::Entry, HashMap, HashSet}, path::PathBuf, sync::{Arc, Mutex}, + time::SystemTime, }; -use crate::env::ResolvedPythonEnv; +use crate::{ + env::ResolvedPythonEnv, + fs_cache::{get_cache_from_file, store_cache_in_file}, +}; lazy_static! { - static ref CACHE: FSCache = FSCache::new(None); + static ref CACHE: CacheImpl = CacheImpl::new(None); } pub trait CacheEntry: Send + Sync { fn get(&self) -> Option; - fn store(&self, executable: PathBuf, environment: ResolvedPythonEnv); - fn track_symlinks(&self, executable: PathBuf, symlinks: Option>); + fn store(&self, environment: ResolvedPythonEnv); + fn track_symlinks(&self, symlinks: Vec); } pub fn create_cache(executable: PathBuf) -> Arc>> { @@ -34,120 +37,175 @@ pub fn set_cache_directory(cache_dir: PathBuf) { CACHE.set_cache_directory(cache_dir) } -pub fn generate_hash(executable: &PathBuf) -> String { - let mut hasher = Sha256::new(); - hasher.update(norm_case(executable).to_string_lossy().as_bytes()); - let h_bytes = hasher.finalize(); - // Convert 256 bits => Hext and then take 16 of the hex chars (that should be unique enough) - // We will handle collisions if they happen. - format!("{:x}", h_bytes)[..16].to_string() -} - pub type LockableCacheEntry = Arc>>; -struct FSCache { +/// Cache of Interpreter details for a given executable. +/// Uses in memory cache as well as a file cache as backing store. +struct CacheImpl { cache_dir: Arc>>, locks: Mutex>, } -impl FSCache { - pub fn new(cache_dir: Option) -> FSCache { - FSCache { +impl CacheImpl { + fn new(cache_dir: Option) -> CacheImpl { + CacheImpl { cache_dir: Arc::new(Mutex::new(cache_dir)), locks: Mutex::new(HashMap::::new()), } } - pub fn get_cache_directory(&self) -> Option { + fn get_cache_directory(&self) -> Option { self.cache_dir.lock().unwrap().clone() } /// Once a cache directory has been set, you cannot change it. /// No point supporting such a scenario. - pub fn set_cache_directory(&self, cache_dir: PathBuf) { + fn set_cache_directory(&self, cache_dir: PathBuf) { self.cache_dir.lock().unwrap().replace(cache_dir); } - pub fn create_cache(&self, executable: PathBuf) -> LockableCacheEntry { - match self.locks.lock().unwrap().entry(executable.clone()) { - Entry::Occupied(lock) => lock.get().clone(), - Entry::Vacant(lock) => { - let cache = Box::new(FSCacheEntry::create()) as Box<(dyn CacheEntry + 'static)>; - lock.insert(Arc::new(Mutex::new(cache))).clone() + fn create_cache(&self, executable: PathBuf) -> LockableCacheEntry { + if let Some(cache_directory) = self.cache_dir.lock().unwrap().as_ref() { + match self.locks.lock().unwrap().entry(executable.clone()) { + Entry::Occupied(lock) => lock.get().clone(), + Entry::Vacant(lock) => { + let cache = + Box::new(CacheEntryImpl::create(cache_directory.clone(), executable)) + as Box<(dyn CacheEntry + 'static)>; + lock.insert(Arc::new(Mutex::new(cache))).clone() + } } + } else { + Arc::new(Mutex::new(Box::new(CacheEntryImpl::empty( + executable.clone(), + )))) } } } -struct FSCacheEntry { +type FilePathWithMTimeCTime = (PathBuf, Option, Option); + +struct CacheEntryImpl { + cache_directory: Option, + executable: PathBuf, envoronment: Arc>>, + /// List of known symlinks to this executable. + symlinks: Arc>>, } -impl FSCacheEntry { - pub fn create() -> impl CacheEntry { - FSCacheEntry { +impl CacheEntryImpl { + pub fn create(cache_directory: PathBuf, executable: PathBuf) -> impl CacheEntry { + CacheEntryImpl { + cache_directory: Some(cache_directory), + executable, + envoronment: Arc::new(Mutex::new(None)), + symlinks: Arc::new(Mutex::new(Vec::new())), + } + } + pub fn empty(executable: PathBuf) -> impl CacheEntry { + CacheEntryImpl { + cache_directory: None, + executable, envoronment: Arc::new(Mutex::new(None)), + symlinks: Arc::new(Mutex::new(Vec::new())), + } + } + pub fn verify_in_memory_cache(&self) { + // Check if any of the exes have changed since we last cached this. + for symlink_info in self.symlinks.lock().unwrap().iter() { + if let Ok(metadata) = symlink_info.0.metadata() { + if metadata.modified().ok() != symlink_info.1 + || metadata.created().ok() != symlink_info.2 + { + self.envoronment.lock().unwrap().take(); + } + } } } } -impl CacheEntry for FSCacheEntry { +impl CacheEntry for CacheEntryImpl { fn get(&self) -> Option { - self.envoronment.lock().unwrap().clone() - } + self.verify_in_memory_cache(); - fn store(&self, _executable: PathBuf, environment: ResolvedPythonEnv) { - self.envoronment.lock().unwrap().replace(environment); - } + // New scope to drop lock immediately after we have the value. + { + if let Some(env) = self.envoronment.lock().unwrap().clone() { + return Some(env); + } + } - fn track_symlinks(&self, _executable: PathBuf, _symlinks: Option>) { - todo!() + if let Some(ref cache_directory) = self.cache_directory { + let (env, symlinks) = get_cache_from_file(cache_directory, &self.executable)?; + self.envoronment.lock().unwrap().replace(env.clone()); + self.symlinks.lock().unwrap().clear(); + self.symlinks.lock().unwrap().append(&mut symlinks.clone()); + Some(env) + } else { + None + } } -} -#[cfg(test)] -mod tests { - use super::*; - - #[test] - #[cfg(unix)] - fn test_hash_generation() { - assert_eq!( - generate_hash(&PathBuf::from( - "/Users/donjayamanne/demo/.venvTestInstall1/bin/python3.12" - )), - "e72c82125e7281e2" - ); - } + fn store(&self, environment: ResolvedPythonEnv) { + // Get hold of the mtimes and ctimes of the symlinks. + let mut symlinks = vec![]; + for symlink in environment.symlinks.clone().unwrap_or_default().iter() { + if let Ok(metadata) = symlink.metadata() { + symlinks.push(( + symlink.clone(), + metadata.modified().ok(), + metadata.created().ok(), + )); + } + } + + symlinks.sort(); + symlinks.dedup(); + + self.symlinks.lock().unwrap().clear(); + self.symlinks.lock().unwrap().append(&mut symlinks.clone()); + self.envoronment + .lock() + .unwrap() + .replace(environment.clone()); - #[test] - #[cfg(unix)] - fn test_hash_generation_upper_case() { - assert_eq!( - generate_hash(&PathBuf::from( - "/Users/donjayamanne/DEMO/.venvTestInstall1/bin/python3.12" - )), - "ecb0ee73d6ddfe97" - ); + if let Some(ref cache_directory) = self.cache_directory { + trace!("Storing cache for {:?}", self.executable); + store_cache_in_file(cache_directory, &self.executable, &environment, symlinks) + } } - // #[test] - // #[cfg(unix)] - // fn test_hash_generation_upper_case() { - // let hashed_name = generate_env_name( - // "new-project", - // &"/Users/donjayamanne/temp/POETRY-UPPER/new-PROJECT".into(), - // ); - - // assert_eq!(hashed_name, "new-project-TbBV0MKD-py"); - // } - - // #[test] - // #[cfg(windows)] - // fn test_hash_generation_windows() { - // let hashed_name = generate_env_name( - // "demo-project1", - // &"C:\\temp\\poetry-folders\\demo-project1".into(), - // ); - - // assert_eq!(hashed_name, "demo-project1-f7sQRtG5-py"); - // } + fn track_symlinks(&self, symlinks: Vec) { + self.verify_in_memory_cache(); + + // If we have already seen this symlink, then we do not need to do anything. + let known_symlinks: HashSet = self + .symlinks + .lock() + .unwrap() + .clone() + .iter() + .map(|x| x.0.clone()) + .collect(); + + if symlinks.iter().all(|x| known_symlinks.contains(x)) { + return; + } + + if let Some(ref cache_directory) = self.cache_directory { + if let Some((mut env, _)) = get_cache_from_file(cache_directory, &self.executable) { + let mut all_symlinks = vec![]; + all_symlinks.append(&mut env.symlinks.clone().unwrap_or_default()); + all_symlinks.append(&mut symlinks.clone()); + all_symlinks.sort(); + all_symlinks.dedup(); + + // Chech whether the details in the cache are the same as the ones we are about to cache. + + env.symlinks = Some(all_symlinks); + trace!("Updating cache for {:?} with new symlinks", self.executable); + self.store(env); + } else { + // Unlikely scenario. + } + } + } } diff --git a/crates/pet-python-utils/src/env.rs b/crates/pet-python-utils/src/env.rs index 94e10713..842fb02c 100644 --- a/crates/pet-python-utils/src/env.rs +++ b/crates/pet-python-utils/src/env.rs @@ -2,8 +2,8 @@ // Licensed under the MIT License. use log::{error, trace}; -use pet_core::env::PythonEnv; -use serde::Deserialize; +use pet_core::{arch::Architecture, env::PythonEnv, python_environment::PythonEnvironment}; +use serde::{Deserialize, Serialize}; use std::{ path::{Path, PathBuf}, time::SystemTime, @@ -22,7 +22,8 @@ pub struct InterpreterInfo { pub is64_bit: bool, } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] pub struct ResolvedPythonEnv { pub executable: PathBuf, pub prefix: PathBuf, @@ -41,6 +42,26 @@ impl ResolvedPythonEnv { env.symlinks.clone_from(&self.symlinks); env } + pub fn add_to_cache(&self, environment: PythonEnvironment) { + // Verify whether we have been given the right exe. + let arch = Some(if self.is64_bit { + Architecture::X64 + } else { + Architecture::X86 + }); + let symlinks = environment.symlinks.clone().unwrap_or_default(); + if symlinks.contains(&self.executable) + && environment.version.clone().unwrap_or_default() == self.version + && environment.prefix.clone().unwrap_or_default() == self.prefix + && environment.arch == arch + { + let cache = create_cache(self.executable.clone()); + let entry = cache.lock().unwrap(); + entry.track_symlinks(symlinks) + } else { + error!("Invalid Python environment being cached: {:?}", environment); + } + } /// Given the executable path, resolve the python environment by spawning python. /// If we had previously spawned Python and we have the symlinks to this as well, /// & all of them are the same as when this exe was previously spawned, @@ -55,7 +76,7 @@ impl ResolvedPythonEnv { if let Some(env) = entry.get() { Some(env) } else if let Some(env) = get_interpreter_details(executable) { - entry.store(executable.to_path_buf(), env.clone()); + entry.store(env.clone()); Some(env) } else { None @@ -82,16 +103,18 @@ fn get_interpreter_details(executable: &Path) -> Option { ); if let Some((_, output)) = output.split_once(PYTHON_INFO_JSON_SEPARATOR) { if let Ok(info) = serde_json::from_str::(output) { + let mut symlinks = vec![ + PathBuf::from(executable), + PathBuf::from(info.executable.clone()), + ]; + symlinks.sort(); + symlinks.dedup(); Some(ResolvedPythonEnv { executable: PathBuf::from(info.executable.clone()), prefix: PathBuf::from(info.sys_prefix), version: info.version.trim().to_string(), is64_bit: info.is64_bit, - symlinks: if info.executable == executable { - None - } else { - Some(vec![PathBuf::from(executable)]) - }, + symlinks: Some(symlinks), }) } else { error!( diff --git a/crates/pet-python-utils/src/fs_cache.rs b/crates/pet-python-utils/src/fs_cache.rs new file mode 100644 index 00000000..1152cc59 --- /dev/null +++ b/crates/pet-python-utils/src/fs_cache.rs @@ -0,0 +1,122 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use log::trace; +use pet_fs::path::norm_case; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::{ + fs::{self, File}, + io::BufReader, + path::{Path, PathBuf}, + time::SystemTime, +}; + +use crate::env::ResolvedPythonEnv; + +type FilePathWithMTimeCTime = (PathBuf, Option, Option); + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +struct CacheEntry { + pub environment: ResolvedPythonEnv, + pub symlinks: Vec, +} + +fn generate_cache_file(cache_directory: &Path, executable: &PathBuf) -> PathBuf { + cache_directory.join(format!("{}.1.json", generate_hash(executable))) +} + +pub fn get_cache_from_file( + cache_directory: &Path, + executable: &PathBuf, +) -> Option<(ResolvedPythonEnv, Vec)> { + let cache_file = generate_cache_file(cache_directory, executable); + let file = File::open(cache_file.clone()).ok()?; + let reader = BufReader::new(file); + let cache: CacheEntry = serde_json::from_reader(reader).ok()?; + + // Check if any of the exes have changed since we last cached them. + let cache_is_valid = cache.symlinks.iter().all(|symlink| { + if let Ok(metadata) = symlink.0.metadata() { + metadata.modified().ok() == symlink.1 && metadata.created().ok() == symlink.2 + } else { + // File may have been deleted. + false + } + }); + + if cache_is_valid { + trace!("Using cache from {:?} for {:?}", cache_file, executable); + Some((cache.environment, cache.symlinks)) + } else { + let _ = fs::remove_file(cache_file); + None + } +} + +pub fn store_cache_in_file( + cache_directory: &Path, + executable: &PathBuf, + environment: &ResolvedPythonEnv, + symlinks_with_times: Vec, +) { + let cache_file = generate_cache_file(cache_directory, executable); + let _ = std::fs::create_dir_all(cache_directory); + + let cache = CacheEntry { + environment: environment.clone(), + symlinks: symlinks_with_times, + }; + if let Ok(file) = std::fs::File::create(cache_file.clone()) { + trace!("Caching {:?} in {:?}", executable, cache_file); + let _ = serde_json::to_writer_pretty(file, &cache).ok(); + } +} + +fn generate_hash(executable: &PathBuf) -> String { + let mut hasher = Sha256::new(); + hasher.update(norm_case(executable).to_string_lossy().as_bytes()); + let h_bytes = hasher.finalize(); + // Convert 256 bits => Hext and then take 16 of the hex chars (that should be unique enough) + // We will handle collisions if they happen. + format!("{:x}", h_bytes)[..16].to_string() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + #[cfg(unix)] + fn test_hash_generation() { + assert_eq!( + generate_hash(&PathBuf::from( + "/Users/donjayamanne/demo/.venvTestInstall1/bin/python3.12" + )), + "e72c82125e7281e2" + ); + } + + #[test] + #[cfg(unix)] + fn test_hash_generation_upper_case() { + assert_eq!( + generate_hash(&PathBuf::from( + "/Users/donjayamanne/DEMO/.venvTestInstall1/bin/python3.12" + )), + "ecb0ee73d6ddfe97" + ); + } + + #[test] + #[cfg(windows)] + fn test_hash_generation() { + assert_eq!( + generate_hash(&PathBuf::from( + &"C:\\temp\\poetry-folders\\demo-project1".into(), + )), + "e72c82125e7281e2" + ); + } +} diff --git a/crates/pet-python-utils/src/lib.rs b/crates/pet-python-utils/src/lib.rs index 9816f24d..9a6aedfa 100644 --- a/crates/pet-python-utils/src/lib.rs +++ b/crates/pet-python-utils/src/lib.rs @@ -4,6 +4,7 @@ pub mod cache; pub mod env; pub mod executable; +mod fs_cache; mod headers; pub mod platform_dirs; pub mod version; diff --git a/crates/pet/src/lib.rs b/crates/pet/src/lib.rs index 2f39d0f8..3f31accd 100644 --- a/crates/pet/src/lib.rs +++ b/crates/pet/src/lib.rs @@ -14,6 +14,7 @@ use pet_core::{os_environment::EnvironmentApi, reporter::Reporter, Configuration use pet_env_var_path::get_search_paths_from_env_variables; use pet_poetry::Poetry; use pet_poetry::PoetryLocator; +use pet_python_utils::cache::set_cache_directory; use pet_reporter::collect; use pet_reporter::{self, cache::CacheReporter, stdio}; use resolve::resolve_environment; @@ -33,6 +34,7 @@ pub struct FindOptions { pub workspace_dirs: Option>, pub workspace_only: bool, pub global_only: bool, + pub cache_directory: Option, } pub fn find_and_report_envs_stdio(options: FindOptions) { @@ -49,6 +51,10 @@ pub fn find_and_report_envs_stdio(options: FindOptions) { } else { None }; + + if let Some(cache_directory) = options.cache_directory.clone() { + set_cache_directory(cache_directory); + } let (config, executable_to_find) = create_config(&options); let environment = EnvironmentApi::new(); let conda_locator = Arc::new(Conda::from(&environment)); @@ -220,13 +226,18 @@ fn find_env( } } -pub fn resolve_report_stdio(executable: PathBuf, verbose: bool) { +pub fn resolve_report_stdio(executable: PathBuf, verbose: bool, cache_directory: Option) { stdio::initialize_logger(if verbose { log::LevelFilter::Trace } else { log::LevelFilter::Warn }); let now = SystemTime::now(); + + if let Some(cache_directory) = cache_directory.clone() { + set_cache_directory(cache_directory); + } + let stdio_reporter = Arc::new(stdio::create_reporter(true)); let reporter = CacheReporter::new(stdio_reporter.clone()); let environment = EnvironmentApi::new(); diff --git a/crates/pet/src/main.rs b/crates/pet/src/main.rs index 26f473ca..f8eee2ad 100644 --- a/crates/pet/src/main.rs +++ b/crates/pet/src/main.rs @@ -31,6 +31,10 @@ enum Commands { #[arg(short, long)] list: bool, + /// Directory to cache the environment information after spawning Python. + #[arg(short, long)] + cache_directory: Option, + /// Display verbose output (defaults to warnings). #[arg(short, long)] verbose: bool, @@ -55,6 +59,10 @@ enum Commands { #[arg(value_name = "PYTHON EXE")] executable: PathBuf, + /// Directory to cache the environment information after spawning Python. + #[arg(short, long)] + cache_directory: Option, + /// Whether to display verbose output (defaults to warnings). #[arg(short, long)] verbose: bool, @@ -73,6 +81,7 @@ fn main() { workspace_dirs: None, workspace_only: false, global_only: false, + cache_directory: None, }) { Commands::Find { list, @@ -81,6 +90,7 @@ fn main() { workspace_dirs, workspace_only, global_only, + cache_directory, } => find_and_report_envs_stdio(FindOptions { print_list: list, print_summary: true, @@ -89,11 +99,13 @@ fn main() { workspace_dirs, workspace_only, global_only, + cache_directory, }), Commands::Resolve { executable, verbose, - } => resolve_report_stdio(executable, verbose), + cache_directory, + } => resolve_report_stdio(executable, verbose, cache_directory), Commands::Server => start_jsonrpc_server(), } } diff --git a/crates/pet/src/resolve.rs b/crates/pet/src/resolve.rs index af716121..c99c1675 100644 --- a/crates/pet/src/resolve.rs +++ b/crates/pet/src/resolve.rs @@ -50,8 +50,8 @@ pub fn resolve_environment( symlinks.sort(); symlinks.dedup(); - let version = Some(info.version); - let prefix = Some(info.prefix); + let version = Some(info.version.clone()); + let prefix = Some(info.prefix.clone()); let arch = Some(if info.is64_bit { Architecture::X64 } else { @@ -61,7 +61,7 @@ pub fn resolve_environment( let resolved = PythonEnvironmentBuilder::new(env.kind) .arch(arch) .display_name(env.display_name) - .executable(Some(info.executable)) + .executable(Some(info.executable.clone())) .manager(env.manager) .name(env.name) .prefix(prefix) @@ -70,6 +70,8 @@ pub fn resolve_environment( .version(version) .build(); + info.add_to_cache(resolved.clone()); + Some(ResolvedEnvironment { discovered, resolved: Some(resolved), From 43c6cc7a9359556ce4372b0e9b6de73af9f80acb Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Tue, 16 Jul 2024 11:35:32 +1000 Subject: [PATCH 2/6] oops --- crates/pet-python-utils/src/fs_cache.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/pet-python-utils/src/fs_cache.rs b/crates/pet-python-utils/src/fs_cache.rs index 1152cc59..c4278e32 100644 --- a/crates/pet-python-utils/src/fs_cache.rs +++ b/crates/pet-python-utils/src/fs_cache.rs @@ -114,7 +114,7 @@ mod tests { fn test_hash_generation() { assert_eq!( generate_hash(&PathBuf::from( - &"C:\\temp\\poetry-folders\\demo-project1".into(), + "C:\\temp\\poetry-folders\\demo-project1".to_string(), )), "e72c82125e7281e2" ); From 1ffa58ce59a3ccdd32f94851bc2fdcdceab34aaa Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Tue, 16 Jul 2024 11:39:06 +1000 Subject: [PATCH 3/6] fix windows tests --- crates/pet-python-utils/src/fs_cache.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/crates/pet-python-utils/src/fs_cache.rs b/crates/pet-python-utils/src/fs_cache.rs index c4278e32..6df95387 100644 --- a/crates/pet-python-utils/src/fs_cache.rs +++ b/crates/pet-python-utils/src/fs_cache.rs @@ -116,7 +116,18 @@ mod tests { generate_hash(&PathBuf::from( "C:\\temp\\poetry-folders\\demo-project1".to_string(), )), - "e72c82125e7281e2" + "c3694bfb39d7065b" + ); + } + + #[test] + #[cfg(windows)] + fn test_hash_generation_upper_case() { + assert_eq!( + generate_hash(&PathBuf::from( + "C:\\TEMP\\POETRY-FOLDERS\\demo-project1".to_string(), + )), + "c3694bfb39d7065b" ); } } From 7c88df421f556dd41ce71f03c46fd1daf8c94870 Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Tue, 16 Jul 2024 12:30:29 +1000 Subject: [PATCH 4/6] Fix tests --- crates/pet-poetry/src/lib.rs | 2 ++ crates/pet-pyenv/src/lib.rs | 13 ++++++------- crates/pet-python-utils/src/fs_cache.rs | 11 ----------- crates/pet/tests/ci_test.rs | 1 + 4 files changed, 9 insertions(+), 18 deletions(-) diff --git a/crates/pet-poetry/src/lib.rs b/crates/pet-poetry/src/lib.rs index 7610fe6e..85f6f215 100644 --- a/crates/pet-poetry/src/lib.rs +++ b/crates/pet-poetry/src/lib.rs @@ -3,6 +3,7 @@ use env_variables::EnvVariables; use environment_locations::list_environments; +use log::trace; use manager::PoetryManager; use pet_core::{ env::PythonEnv, @@ -69,6 +70,7 @@ impl Poetry { self.poetry_executable.lock().unwrap().clone(), &self.env_vars, ); + trace!("Poetry Manager {:?}", manager); let mut result = LocatorResult { managers: vec![], environments: vec![], diff --git a/crates/pet-pyenv/src/lib.rs b/crates/pet-pyenv/src/lib.rs index 87fb5882..c7c4dea2 100644 --- a/crates/pet-pyenv/src/lib.rs +++ b/crates/pet-pyenv/src/lib.rs @@ -13,6 +13,7 @@ use std::{ use env_variables::EnvVariables; use environments::{get_generic_python_environment, get_virtual_env_environment}; +use log::trace; use manager::PyEnvInfo; use pet_conda::{utils::is_conda_env, CondaLocator}; use pet_core::{ @@ -33,7 +34,6 @@ mod manager; pub struct PyEnv { pub env_vars: EnvVariables, pub conda_locator: Arc, - found_pyenv: AtomicBool, manager: Arc>>, versions_dir: Arc>>, } @@ -44,7 +44,6 @@ impl PyEnv { conda_locator: Arc, ) -> impl Locator { PyEnv { - found_pyenv: AtomicBool::new(false), env_vars: EnvVariables::from(environment), conda_locator, manager: Arc::new(Mutex::new(None)), @@ -52,16 +51,15 @@ impl PyEnv { } } fn clear(&self) { - self.found_pyenv - .store(false, std::sync::atomic::Ordering::Relaxed); self.manager.lock().unwrap().take(); self.versions_dir.lock().unwrap().take(); } fn get_manager_versions_dir(&self) -> (Option, Option) { let mut managers = self.manager.lock().unwrap(); let mut versions = self.versions_dir.lock().unwrap(); - if !self.found_pyenv.load(Ordering::Relaxed) && (managers.is_none() || versions.is_none()) { + if managers.is_none() || versions.is_none() { let pyenv_info = PyEnvInfo::from(&self.env_vars); + trace!("PyEnv Info {:?}", pyenv_info); if let Some(ref exe) = pyenv_info.exe { let version = pyenv_info.version.clone(); let manager = EnvManager::new(exe.clone(), EnvManagerType::Pyenv, version); @@ -74,9 +72,8 @@ impl PyEnv { } else { versions.take(); } - self.found_pyenv.store(true, Ordering::Relaxed); } - + trace!("PyEnv Manager {:?} and versions {:?}", managers, versions); (managers.clone(), versions.clone()) } } @@ -164,6 +161,8 @@ impl Locator for PyEnv { } } }); + } else { + trace!("PyEnv versions directory not found"); } } } diff --git a/crates/pet-python-utils/src/fs_cache.rs b/crates/pet-python-utils/src/fs_cache.rs index 6df95387..2d359420 100644 --- a/crates/pet-python-utils/src/fs_cache.rs +++ b/crates/pet-python-utils/src/fs_cache.rs @@ -119,15 +119,4 @@ mod tests { "c3694bfb39d7065b" ); } - - #[test] - #[cfg(windows)] - fn test_hash_generation_upper_case() { - assert_eq!( - generate_hash(&PathBuf::from( - "C:\\TEMP\\POETRY-FOLDERS\\demo-project1".to_string(), - )), - "c3694bfb39d7065b" - ); - } } diff --git a/crates/pet/tests/ci_test.rs b/crates/pet/tests/ci_test.rs index 8cf9ba4c..02a6b309 100644 --- a/crates/pet/tests/ci_test.rs +++ b/crates/pet/tests/ci_test.rs @@ -226,6 +226,7 @@ fn check_if_pyenv_virtualenv_exists() { let environment = EnvironmentApi::new(); let conda_locator = Arc::new(Conda::from(&environment)); let poetry_locator = Arc::new(Poetry::from(&environment)); + trace!("Checking for pyenv-virtualenv"); find_and_report_envs( &CacheReporter::new(reporter.clone()), From cadb3afee9cd2740fee595cbab469a962affff48 Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Tue, 16 Jul 2024 12:35:08 +1000 Subject: [PATCH 5/6] Fixes --- crates/pet-pyenv/src/lib.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/crates/pet-pyenv/src/lib.rs b/crates/pet-pyenv/src/lib.rs index c7c4dea2..741e89d6 100644 --- a/crates/pet-pyenv/src/lib.rs +++ b/crates/pet-pyenv/src/lib.rs @@ -4,10 +4,7 @@ use std::{ fs, path::PathBuf, - sync::{ - atomic::{AtomicBool, Ordering}, - Arc, Mutex, - }, + sync::{Arc, Mutex}, thread, }; @@ -73,7 +70,6 @@ impl PyEnv { versions.take(); } } - trace!("PyEnv Manager {:?} and versions {:?}", managers, versions); (managers.clone(), versions.clone()) } } From 5bfe827f801e595ba351ec1cb2d3ba7ca5a0bbbc Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Tue, 16 Jul 2024 12:50:42 +1000 Subject: [PATCH 6/6] logging --- crates/pet/src/find.rs | 6 ++++++ crates/pet/tests/ci_poetry.rs | 5 +---- crates/pet/tests/ci_test.rs | 5 +---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/crates/pet/src/find.rs b/crates/pet/src/find.rs index 457a5e18..6b0bdfc8 100644 --- a/crates/pet/src/find.rs +++ b/crates/pet/src/find.rs @@ -78,7 +78,13 @@ pub fn find_and_report_envs( let summary = summary.clone(); s.spawn(move || { let start = std::time::Instant::now(); + trace!("Searching using locator: {}", locator.get_name()); locator.find(reporter); + trace!( + "Completed searching using locator: {} in {:?}", + locator.get_name(), + start.elapsed() + ); summary .lock() .unwrap() diff --git a/crates/pet/tests/ci_poetry.rs b/crates/pet/tests/ci_poetry.rs index 95bf098f..ba95fe52 100644 --- a/crates/pet/tests/ci_poetry.rs +++ b/crates/pet/tests/ci_poetry.rs @@ -2,10 +2,7 @@ // Licensed under the MIT License. use pet_poetry::Poetry; -use pet_reporter::{ - cache::{self, CacheReporter}, - collect, -}; +use pet_reporter::{cache::CacheReporter, collect}; mod common; diff --git a/crates/pet/tests/ci_test.rs b/crates/pet/tests/ci_test.rs index 02a6b309..306afa75 100644 --- a/crates/pet/tests/ci_test.rs +++ b/crates/pet/tests/ci_test.rs @@ -17,10 +17,7 @@ use pet_core::{ }; use pet_env_var_path::get_search_paths_from_env_variables; use pet_poetry::Poetry; -use pet_reporter::{ - cache::{self, CacheReporter}, - collect, -}; +use pet_reporter::{cache::CacheReporter, collect}; use regex::Regex; use serde::Deserialize;