From 4880dd8db3dafc1b74c622695816e07994b536d2 Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Thu, 11 Jul 2024 12:58:29 +1000 Subject: [PATCH 1/3] Add support for finding without resolving --- crates/pet-core/src/lib.rs | 2 +- crates/pet-pyenv/src/lib.rs | 6 +- crates/pet-python-utils/src/executable.rs | 2 +- crates/pet-reporter/src/collect.rs | 42 ++++ crates/pet-reporter/src/lib.rs | 1 + crates/pet-virtualenv/src/lib.rs | 65 +++--- crates/pet/src/find.rs | 245 ++++++++++++---------- crates/pet/src/jsonrpc.rs | 230 +++++++++++++------- crates/pet/src/lib.rs | 194 ++++++++++++++--- crates/pet/src/main.rs | 54 ++++- crates/pet/tests/ci_homebrew_container.rs | 1 + crates/pet/tests/ci_jupyter_container.rs | 1 + crates/pet/tests/ci_poetry.rs | 4 +- crates/pet/tests/ci_test.rs | 76 ++++++- docs/JSONRPC.md | 30 ++- 15 files changed, 693 insertions(+), 260 deletions(-) create mode 100644 crates/pet-reporter/src/collect.rs diff --git a/crates/pet-core/src/lib.rs b/crates/pet-core/src/lib.rs index 437b0d27..81a20177 100644 --- a/crates/pet-core/src/lib.rs +++ b/crates/pet-core/src/lib.rs @@ -15,7 +15,7 @@ pub mod python_environment; pub mod reporter; pub mod telemetry; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct LocatorResult { pub managers: Vec, pub environments: Vec, diff --git a/crates/pet-pyenv/src/lib.rs b/crates/pet-pyenv/src/lib.rs index e460c616..2a2279a7 100644 --- a/crates/pet-pyenv/src/lib.rs +++ b/crates/pet-pyenv/src/lib.rs @@ -128,12 +128,12 @@ impl Locator for PyEnv { } if let Some(ref versions) = &pyenv_info.versions { if let Some(envs) = list_pyenv_environments(&manager, versions, &self.conda_locator) { - for env in envs.environments { - reporter.report_environment(&env); - } for mgr in envs.managers { reporter.report_manager(&mgr); } + for env in envs.environments { + reporter.report_environment(&env); + } } } } diff --git a/crates/pet-python-utils/src/executable.rs b/crates/pet-python-utils/src/executable.rs index 49c88772..569fa44c 100644 --- a/crates/pet-python-utils/src/executable.rs +++ b/crates/pet-python-utils/src/executable.rs @@ -92,7 +92,7 @@ pub fn find_executables>(env_path: T) -> Vec { python_executables } -fn is_python_executable_name(exe: &Path) -> bool { +pub fn is_python_executable_name(exe: &Path) -> bool { let name = exe .file_name() .unwrap_or_default() diff --git a/crates/pet-reporter/src/collect.rs b/crates/pet-reporter/src/collect.rs new file mode 100644 index 00000000..86c33df9 --- /dev/null +++ b/crates/pet-reporter/src/collect.rs @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use pet_core::{manager::EnvManager, python_environment::PythonEnvironment, reporter::Reporter}; +use std::sync::{Arc, Mutex}; + +/// Used to just collect the environments and managers and will not report anytihng anywhere. +pub struct CollectReporter { + pub managers: Arc>>, + pub environments: Arc>>, +} + +impl Default for CollectReporter { + fn default() -> Self { + Self::new() + } +} + +impl CollectReporter { + pub fn new() -> CollectReporter { + CollectReporter { + managers: Arc::new(Mutex::new(vec![])), + environments: Arc::new(Mutex::new(vec![])), + } + } +} +impl Reporter for CollectReporter { + fn report_telemetry(&self, _event: &pet_core::telemetry::TelemetryEvent) { + // + } + fn report_manager(&self, manager: &EnvManager) { + self.managers.lock().unwrap().push(manager.clone()); + } + + fn report_environment(&self, env: &PythonEnvironment) { + self.environments.lock().unwrap().push(env.clone()); + } +} + +pub fn create_reporter() -> CollectReporter { + CollectReporter::new() +} diff --git a/crates/pet-reporter/src/lib.rs b/crates/pet-reporter/src/lib.rs index 16c42605..31fb37b4 100644 --- a/crates/pet-reporter/src/lib.rs +++ b/crates/pet-reporter/src/lib.rs @@ -2,6 +2,7 @@ // Licensed under the MIT License. pub mod cache; +pub mod collect; pub mod environment; pub mod jsonrpc; pub mod stdio; diff --git a/crates/pet-virtualenv/src/lib.rs b/crates/pet-virtualenv/src/lib.rs index d67d3c30..a836f2cc 100644 --- a/crates/pet-virtualenv/src/lib.rs +++ b/crates/pet-virtualenv/src/lib.rs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +use std::path::Path; + use pet_core::{ python_environment::{PythonEnvironment, PythonEnvironmentBuilder, PythonEnvironmentKind}, reporter::Reporter, @@ -20,34 +22,49 @@ pub fn is_virtualenv(env: &PythonEnv) -> bool { } } if let Some(bin) = env.executable.parent() { - // Check if there are any activate.* files in the same directory as the interpreter. - // - // env - // |__ activate, activate.* <--- check if any of these files exist - // |__ python <--- interpreterPath + return is_virtualenv_dir(bin); + } + + false +} - // if let Some(parent_path) = PathBuf::from(env.) - // const directory = path.dirname(interpreterPath); - // const files = await fsapi.readdir(directory); - // const regex = /^activate(\.([A-z]|\d)+)?$/i; - if bin.join("activate").exists() || bin.join("activate.bat").exists() { - return true; +pub fn is_virtualenv_dir(path: &Path) -> bool { + // Check if the executable is in a bin or Scripts directory. + // Possible for some reason we do not have the prefix. + let mut path = path.to_path_buf(); + if !path.ends_with("bin") && !path.ends_with("Scripts") { + if cfg!(windows) { + path = path.join("Scripts"); + } else { + path = path.join("bin"); } + } + // Check if there are any activate.* files in the same directory as the interpreter. + // + // env + // |__ activate, activate.* <--- check if any of these files exist + // |__ python <--- interpreterPath - // Support for activate.ps, etc. - if let Ok(files) = std::fs::read_dir(bin) { - for file in files.filter_map(Result::ok).map(|e| e.path()) { - if file - .file_name() - .unwrap_or_default() - .to_str() - .unwrap_or_default() - .starts_with("activate") - { - return true; - } + // if let Some(parent_path) = PathBuf::from(env.) + // const directory = path.dirname(interpreterPath); + // const files = await fsapi.readdir(directory); + // const regex = /^activate(\.([A-z]|\d)+)?$/i; + if path.join("activate").exists() || path.join("activate.bat").exists() { + return true; + } + + // Support for activate.ps, etc. + if let Ok(files) = std::fs::read_dir(path) { + for file in files.filter_map(Result::ok).map(|e| e.path()) { + if file + .file_name() + .unwrap_or_default() + .to_str() + .unwrap_or_default() + .starts_with("activate") + { + return true; } - return false; } } diff --git a/crates/pet/src/find.rs b/crates/pet/src/find.rs index b998efe2..c4b63e1f 100644 --- a/crates/pet/src/find.rs +++ b/crates/pet/src/find.rs @@ -2,6 +2,7 @@ // Licensed under the MIT License. use log::{trace, warn}; +use pet_conda::utils::is_conda_env; use pet_core::os_environment::Environment; use pet_core::reporter::Reporter; use pet_core::{Configuration, Locator}; @@ -11,7 +12,8 @@ use pet_python_utils::env::PythonEnv; use pet_python_utils::executable::{ find_executable, find_executables, should_search_for_environments_in_path, }; -use pet_venv::is_venv_dir; +use pet_virtualenv::is_virtualenv_dir; +use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; use std::fs; use std::path::PathBuf; @@ -30,11 +32,21 @@ pub struct Summary { pub find_workspace_directories_time: Duration, } +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum SearchScope { + /// Search for environments in global space. + Global, + /// Search for environments in workspace folder. + Workspace, +} + pub fn find_and_report_envs( reporter: &dyn Reporter, configuration: Configuration, locators: &Arc>>, environment: &dyn Environment, + search_scope: Option, ) -> Arc> { let summary = Arc::new(Mutex::new(Summary { time: Duration::from_secs(0), @@ -49,74 +61,91 @@ pub fn find_and_report_envs( // From settings let environment_directories = configuration.environment_directories.unwrap_or_default(); let workspace_directories = configuration.workspace_directories.unwrap_or_default(); + let search_global = match search_scope { + Some(SearchScope::Global) => true, + Some(SearchScope::Workspace) => false, + _ => true, + }; + let search_workspace = match search_scope { + Some(SearchScope::Global) => false, + Some(SearchScope::Workspace) => true, + _ => true, + }; + thread::scope(|s| { // 1. Find using known global locators. s.spawn(|| { // Find in all the finders let start = std::time::Instant::now(); - thread::scope(|s| { - for locator in locators.iter() { - let locator = locator.clone(); - let summary = summary.clone(); - s.spawn(move || { - let start = std::time::Instant::now(); - locator.find(reporter); - summary - .lock() - .unwrap() - .find_locators_times - .insert(locator.get_name(), start.elapsed()); - }); - } - }); + if search_global { + thread::scope(|s| { + for locator in locators.iter() { + let locator = locator.clone(); + let summary = summary.clone(); + s.spawn(move || { + let start = std::time::Instant::now(); + locator.find(reporter); + summary + .lock() + .unwrap() + .find_locators_times + .insert(locator.get_name(), start.elapsed()); + }); + } + }); + } summary.lock().unwrap().find_locators_time = start.elapsed(); }); // Step 2: Search in PATH variable s.spawn(|| { let start = std::time::Instant::now(); - let global_env_search_paths: Vec = - get_search_paths_from_env_variables(environment); + if search_global { + let global_env_search_paths: Vec = + get_search_paths_from_env_variables(environment); - trace!( - "Searching for environments in global folders: {:?}", - global_env_search_paths - ); - find_python_environments( - global_env_search_paths.clone(), - reporter, - locators, - false, - &global_env_search_paths, - ); + trace!( + "Searching for environments in global folders: {:?}", + global_env_search_paths + ); + find_python_environments( + global_env_search_paths.clone(), + reporter, + locators, + false, + &global_env_search_paths, + ); + } summary.lock().unwrap().find_path_time = start.elapsed(); }); // Step 3: Search in some global locations for virtual envs. s.spawn(|| { let start = std::time::Instant::now(); - let search_paths: Vec = [ - list_global_virtual_envs_paths( - environment.get_env_var("WORKON_HOME".into()), - environment.get_env_var("XDG_DATA_HOME".into()), - environment.get_user_home(), - ), - environment_directories, - ] - .concat(); - let global_env_search_paths: Vec = - get_search_paths_from_env_variables(environment); + if search_global { + let search_paths: Vec = [ + list_global_virtual_envs_paths( + environment.get_env_var("WORKON_HOME".into()), + environment.get_env_var("XDG_DATA_HOME".into()), + environment.get_user_home(), + ), + environment_directories, + ] + .concat(); + let global_env_search_paths: Vec = + get_search_paths_from_env_variables(environment); - trace!( - "Searching for environments in global venv folders: {:?}", - search_paths - ); + trace!( + "Searching for environments in global venv folders: {:?}", + search_paths + ); - find_python_environments( - search_paths, - reporter, - locators, - false, - &global_env_search_paths, - ); + find_python_environments( + search_paths, + reporter, + locators, + false, + &global_env_search_paths, + ); + } summary.lock().unwrap().find_global_virtual_envs_time = start.elapsed(); }); // Step 4: Find in workspace folders too. @@ -127,19 +156,26 @@ pub fn find_and_report_envs( // & users can have a lot of workspace folders and can have a large number fo files/directories // that could the discovery. s.spawn(|| { - if workspace_directories.is_empty() { - return; - } - trace!( - "Searching for environments in custom folders: {:?}", - workspace_directories - ); let start = std::time::Instant::now(); - find_python_environments_in_workspace_folders_recursive( - workspace_directories, - reporter, - locators, - ); + if search_workspace && !workspace_directories.is_empty() { + trace!( + "Searching for environments in workspace folders: {:?}", + workspace_directories + ); + let global_env_search_paths: Vec = + get_search_paths_from_env_variables(environment); + for workspace_folder in workspace_directories { + let global_env_search_paths = global_env_search_paths.clone(); + s.spawn(move || { + find_python_environments_in_workspace_folder_recursive( + &workspace_folder, + reporter, + locators, + &global_env_search_paths, + ); + }); + } + } summary.lock().unwrap().find_workspace_directories_time = start.elapsed(); }); }); @@ -148,50 +184,47 @@ pub fn find_and_report_envs( summary } -fn find_python_environments_in_workspace_folders_recursive( - workspace_folders: Vec, +pub fn find_python_environments_in_workspace_folder_recursive( + workspace_folder: &PathBuf, reporter: &dyn Reporter, locators: &Arc>>, + global_env_search_paths: &[PathBuf], ) { - thread::scope(|s| { - s.spawn(|| { - for workspace_folder in workspace_folders { - let paths_to_search_first = vec![ - // Possible this is a virtual env - workspace_folder.clone(), - // Optimize for finding these first. - workspace_folder.join(".venv"), - workspace_folder.join(".conda"), - workspace_folder.join(".virtualenv"), - workspace_folder.join("venv"), - ]; - find_python_environments_in_paths_with_locators( - paths_to_search_first.clone(), - locators, - reporter, - true, - &[], - ); + // When searching in a directory, give preference to some paths. + let paths_to_search_first = vec![ + // Possible this is a virtual env + workspace_folder.to_path_buf(), + // Optimize for finding these first. + workspace_folder.join(".venv"), + workspace_folder.join(".conda"), + workspace_folder.join(".virtualenv"), + workspace_folder.join("venv"), + ]; - // If this is a virtual env folder, no need to scan this. - if is_venv_dir(&workspace_folder) { - continue; - } + // Possible this is an environment. + find_python_environments_in_paths_with_locators( + paths_to_search_first.clone(), + locators, + reporter, + true, + global_env_search_paths, + ); - if let Ok(reader) = fs::read_dir(&workspace_folder) { - for folder in reader - .filter_map(Result::ok) - .filter(|d| d.file_type().is_ok_and(|f| f.is_dir())) - .map(|p| p.path()) - .filter(should_search_for_environments_in_path) - .filter(|p| !paths_to_search_first.contains(p)) - { - find_python_environments(vec![folder], reporter, locators, true, &[]); - } - } - } - }); - }); + // If this is a virtual env folder, no need to scan this. + if is_virtualenv_dir(workspace_folder) || is_conda_env(workspace_folder) { + return; + } + if let Ok(reader) = fs::read_dir(workspace_folder) { + for folder in reader + .filter_map(Result::ok) + .filter(|d| d.file_type().is_ok_and(|f| f.is_dir())) + .map(|p| p.path()) + .filter(should_search_for_environments_in_path) + .filter(|p| !paths_to_search_first.contains(p)) + { + find_python_environments(vec![folder], reporter, locators, true, &[]); + } + } } fn find_python_environments( @@ -265,7 +298,7 @@ fn find_python_environments_in_paths_with_locators( } } -fn identify_python_executables_using_locators( +pub fn identify_python_executables_using_locators( executables: Vec, locators: &Arc>>, reporter: &dyn Reporter, @@ -277,10 +310,10 @@ fn identify_python_executables_using_locators( if let Some(env) = identify_python_environment_using_locators(&env, locators, global_env_search_paths) { - reporter.report_environment(&env); - if let Some(manager) = env.manager { - reporter.report_manager(&manager); + if let Some(manager) = &env.manager { + reporter.report_manager(manager); } + reporter.report_environment(&env); continue; } else { warn!("Unknown Python Env {:?}", executable); diff --git a/crates/pet/src/jsonrpc.rs b/crates/pet/src/jsonrpc.rs index 0523bc21..5da20c0d 100644 --- a/crates/pet/src/jsonrpc.rs +++ b/crates/pet/src/jsonrpc.rs @@ -5,16 +5,19 @@ use log::{error, info, trace}; use pet::resolve::resolve_environment; use pet_conda::Conda; use pet_conda::CondaLocator; +use pet_core::python_environment::PythonEnvironment; use pet_core::{ os_environment::{Environment, EnvironmentApi}, Configuration, Locator, }; +use pet_env_var_path::get_search_paths_from_env_variables; use pet_jsonrpc::{ send_error, send_reply, server::{start_server, HandlersKeyedByMethodName}, }; use pet_poetry::Poetry; use pet_poetry::PoetryLocator; +use pet_reporter::collect; use pet_reporter::{cache::CacheReporter, jsonrpc}; use pet_telemetry::report_inaccuracies_identified_after_resolving; use serde::{Deserialize, Serialize}; @@ -28,7 +31,11 @@ use std::{ time::{Duration, SystemTime}, }; -use crate::{find::find_and_report_envs, locators::create_locators}; +use crate::find::find_and_report_envs; +use crate::find::find_python_environments_in_workspace_folder_recursive; +use crate::find::identify_python_executables_using_locators; +use crate::find::SearchScope; +use crate::locators::create_locators; pub struct Context { configuration: RwLock, @@ -60,6 +67,7 @@ pub fn start_jsonrpc_server() { handlers.add_request_handler("configure", handle_configure); handlers.add_request_handler("refresh", handle_refresh); handlers.add_request_handler("resolve", handle_resolve); + handlers.add_request_handler("find", handle_find); start_server(&handlers) } @@ -101,6 +109,14 @@ pub fn handle_configure(context: Arc, id: u32, params: Value) { } } +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct RefreshOptions { + /// The search paths are the paths where we will look for environments. + /// Defaults to searching everywhere (or when None), else it can be restricted to a specific scope. + pub search_scope: Option, +} + #[derive(Debug, Clone, Deserialize, Serialize)] pub struct RefreshResult { duration: u128, @@ -114,81 +130,95 @@ impl RefreshResult { } } -pub fn handle_refresh(context: Arc, id: u32, _params: Value) { - // Start in a new thread, we can have multiple requests. - thread::spawn(move || { - let config = context.configuration.read().unwrap().clone(); - let reporter = Arc::new(CacheReporter::new(Arc::new(jsonrpc::create_reporter()))); - - trace!("Start refreshing environments, config: {:?}", config); - let summary = find_and_report_envs( - reporter.as_ref(), - config, - &context.locators, - context.os_environment.deref(), - ); - let summary = summary.lock().unwrap(); - for locator in summary.find_locators_times.iter() { - info!("Locator {} took {:?}", locator.0, locator.1); - } - info!( - "Environments found using locators in {:?}", - summary.find_locators_time - ); - info!("Environments in PATH found in {:?}", summary.find_path_time); - info!( - "Environments in global virtual env paths found in {:?}", - summary.find_global_virtual_envs_time - ); - info!( - "Environments in workspace folders found in {:?}", - summary.find_workspace_directories_time - ); - trace!("Finished refreshing environments in {:?}", summary.time); - send_reply(id, Some(RefreshResult::new(summary.time))); - - // Find an report missing envs for the first launch of this process. - if MISSING_ENVS_REPORTED - .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed) - .ok() - .unwrap_or_default() - { - // By now all conda envs have been found - // Spawn conda in a separate thread. - // & see if we can find more environments by spawning conda. - // But we will not wait for this to complete. - let conda_locator = context.conda_locator.clone(); - let conda_executable = context - .configuration - .read() - .unwrap() - .conda_executable - .clone(); - let reporter_ref = reporter.clone(); +pub fn handle_refresh(context: Arc, id: u32, params: Value) { + match serde_json::from_value::(params.clone()) { + Ok(refres_options) => { + // Start in a new thread, we can have multiple requests. thread::spawn(move || { - conda_locator.find_and_report_missing_envs(reporter_ref.as_ref(), conda_executable); - Some(()) - }); + let config = context.configuration.read().unwrap().clone(); + let reporter = Arc::new(CacheReporter::new(Arc::new(jsonrpc::create_reporter()))); - // By now all poetry envs have been found - // Spawn poetry exe in a separate thread. - // & see if we can find more environments by spawning poetry. - // But we will not wait for this to complete. - let poetry_locator = context.poetry_locator.clone(); - let poetry_executable = context - .configuration - .read() - .unwrap() - .poetry_executable - .clone(); - let reporter_ref = reporter.clone(); - thread::spawn(move || { - poetry_locator - .find_and_report_missing_envs(reporter_ref.as_ref(), poetry_executable); - Some(()) + trace!("Start refreshing environments, config: {:?}", config); + let summary = find_and_report_envs( + reporter.as_ref(), + config, + &context.locators, + context.os_environment.deref(), + refres_options.search_scope, + ); + let summary = summary.lock().unwrap(); + for locator in summary.find_locators_times.iter() { + info!("Locator {} took {:?}", locator.0, locator.1); + } + info!( + "Environments found using locators in {:?}", + summary.find_locators_time + ); + info!("Environments in PATH found in {:?}", summary.find_path_time); + info!( + "Environments in global virtual env paths found in {:?}", + summary.find_global_virtual_envs_time + ); + info!( + "Environments in workspace folders found in {:?}", + summary.find_workspace_directories_time + ); + trace!("Finished refreshing environments in {:?}", summary.time); + send_reply(id, Some(RefreshResult::new(summary.time))); + + // Find an report missing envs for the first launch of this process. + if MISSING_ENVS_REPORTED + .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed) + .ok() + .unwrap_or_default() + { + // By now all conda envs have been found + // Spawn conda in a separate thread. + // & see if we can find more environments by spawning conda. + // But we will not wait for this to complete. + let conda_locator = context.conda_locator.clone(); + let conda_executable = context + .configuration + .read() + .unwrap() + .conda_executable + .clone(); + let reporter_ref = reporter.clone(); + thread::spawn(move || { + conda_locator + .find_and_report_missing_envs(reporter_ref.as_ref(), conda_executable); + Some(()) + }); + + // By now all poetry envs have been found + // Spawn poetry exe in a separate thread. + // & see if we can find more environments by spawning poetry. + // But we will not wait for this to complete. + let poetry_locator = context.poetry_locator.clone(); + let poetry_executable = context + .configuration + .read() + .unwrap() + .poetry_executable + .clone(); + let reporter_ref = reporter.clone(); + thread::spawn(move || { + poetry_locator + .find_and_report_missing_envs(reporter_ref.as_ref(), poetry_executable); + Some(()) + }); + } }); } - }); + Err(e) => { + error!("Failed to parse refresh {params:?}: {e}"); + send_error( + Some(id), + -4, + format!("Failed to parse refresh {params:?}: {e}"), + ); + } + } } #[derive(Debug, Clone, Deserialize, Serialize)] @@ -240,12 +270,64 @@ pub fn handle_resolve(context: Arc, id: u32, params: Value) { }); } Err(e) => { - error!("Failed to parse request {params:?}: {e}"); + error!("Failed to parse resolve {params:?}: {e}"); send_error( Some(id), -4, - format!("Failed to parse request {params:?}: {e}"), + format!("Failed to parse resolve {params:?}: {e}"), ); } } } + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct FindOptions { + /// Search path, can be a directory or a Python executable as well. + /// If passing a directory, the assumption is that its a project directory (workspace folder). + /// This is important, because any poetry/pipenv environment found will have the project directory set. + pub search_path: PathBuf, +} + +pub fn handle_find(context: Arc, id: u32, params: Value) { + thread::spawn( + move || match serde_json::from_value::(params.clone()) { + Ok(find_options) => { + let global_env_search_paths: Vec = + get_search_paths_from_env_variables(context.os_environment.as_ref()); + + let collect_reporter = Arc::new(collect::create_reporter()); + let reporter = CacheReporter::new(collect_reporter.clone()); + if find_options.search_path.is_file() { + identify_python_executables_using_locators( + vec![find_options.search_path.clone()], + &context.locators, + &reporter, + &global_env_search_paths, + ); + } else { + find_python_environments_in_workspace_folder_recursive( + &find_options.search_path, + &reporter, + &context.locators, + &global_env_search_paths, + ); + } + + let envs = collect_reporter.environments.lock().unwrap().clone(); + if envs.is_empty() { + send_reply(id, None::>); + } else { + send_reply(id, envs.into()); + } + } + Err(e) => { + error!("Failed to parse find {params:?}: {e}"); + send_error( + Some(id), + -4, + format!("Failed to parse find {params:?}: {e}"), + ); + } + }, + ); +} diff --git a/crates/pet/src/lib.rs b/crates/pet/src/lib.rs index ee6d12b1..853bd8d8 100644 --- a/crates/pet/src/lib.rs +++ b/crates/pet/src/lib.rs @@ -2,49 +2,120 @@ // Licensed under the MIT License. use find::find_and_report_envs; +use find::identify_python_executables_using_locators; +use find::SearchScope; use locators::create_locators; +use log::warn; use pet_conda::Conda; use pet_conda::CondaLocator; -use pet_core::{os_environment::EnvironmentApi, Configuration}; +use pet_core::os_environment::Environment; +use pet_core::Locator; +use pet_core::{os_environment::EnvironmentApi, reporter::Reporter, Configuration}; +use pet_env_var_path::get_search_paths_from_env_variables; use pet_poetry::Poetry; use pet_poetry::PoetryLocator; +use pet_reporter::collect; use pet_reporter::{self, cache::CacheReporter, stdio}; +use resolve::resolve_environment; +use std::path::PathBuf; use std::{collections::BTreeMap, env, sync::Arc, time::SystemTime}; pub mod find; pub mod locators; pub mod resolve; -pub fn find_and_report_envs_stdio( - print_list: bool, - print_summary: bool, - verbose: bool, - report_missing: bool, -) { - stdio::initialize_logger(if verbose { +#[derive(Debug, Clone)] +pub struct FindOptions { + pub print_list: bool, + pub print_summary: bool, + pub verbose: bool, + pub report_missing: bool, + pub workspace_dirs: Option>, + pub workspace_only: bool, + pub global_only: bool, +} + +pub fn find_and_report_envs_stdio(options: FindOptions) { + stdio::initialize_logger(if options.verbose { log::LevelFilter::Trace } else { - log::LevelFilter::Info + log::LevelFilter::Warn }); let now = SystemTime::now(); - - let stdio_reporter = Arc::new(stdio::create_reporter(print_list)); - let reporter = CacheReporter::new(stdio_reporter.clone()); + let search_scope = if options.workspace_only { + Some(SearchScope::Workspace) + } else if options.global_only { + Some(SearchScope::Global) + } else { + None + }; + let (config, executable_to_find) = create_config(&options); let environment = EnvironmentApi::new(); let conda_locator = Arc::new(Conda::from(&environment)); let poetry_locator = Arc::new(Poetry::from(&environment)); - let mut config = Configuration::default(); - if let Ok(cwd) = env::current_dir() { - config.workspace_directories = Some(vec![cwd]); - } let locators = create_locators(conda_locator.clone(), poetry_locator.clone(), &environment); for locator in locators.iter() { locator.configure(&config); } - let summary = find_and_report_envs(&reporter, config, &locators, &environment); - if report_missing { + if let Some(executable) = executable_to_find { + find_env(&executable, &locators, &environment) + } else { + find_envs( + &options, + &locators, + config, + conda_locator.as_ref(), + poetry_locator.as_ref(), + &environment, + search_scope, + ); + } + + println!("Completed in {}ms", now.elapsed().unwrap().as_millis()) +} + +fn create_config(options: &FindOptions) -> (Configuration, Option) { + let mut config = Configuration::default(); + let mut workspace_directories = vec![]; + if let Some(dirs) = options.workspace_dirs.clone() { + workspace_directories.extend(dirs); + } + // If workspace folders have been provided do not add cwd. + if workspace_directories.is_empty() { + if let Ok(cwd) = env::current_dir() { + workspace_directories.push(cwd); + } + } + workspace_directories.sort(); + workspace_directories.dedup(); + + let executable_to_find = + if workspace_directories.len() == 1 && workspace_directories[0].is_file() { + Some(workspace_directories[0].clone()) + } else { + None + }; + config.workspace_directories = Some(workspace_directories); + + (config, executable_to_find) +} + +fn find_envs( + options: &FindOptions, + locators: &Arc>>, + config: Configuration, + conda_locator: &Conda, + poetry_locator: &Poetry, + environment: &dyn Environment, + search_scope: Option, +) { + let stdio_reporter = Arc::new(stdio::create_reporter(options.print_list)); + let reporter = CacheReporter::new(stdio_reporter.clone()); + + let summary = find_and_report_envs(&reporter, config, locators, environment, search_scope); + if options.report_missing { // By now all conda envs have been found // Spawn conda // & see if we can find more environments by spawning conda. @@ -52,15 +123,17 @@ pub fn find_and_report_envs_stdio( let _ = poetry_locator.find_and_report_missing_envs(&reporter, None); } - if print_summary { + if options.print_summary { let summary = summary.lock().unwrap(); - println!(); - println!("Breakdown by each locator:"); - println!("--------------------------"); - for locator in summary.find_locators_times.iter() { - println!("{:<20} : {:?}", locator.0, locator.1); + if !summary.find_locators_times.is_empty() { + println!(); + println!("Breakdown by each locator:"); + println!("--------------------------"); + for locator in summary.find_locators_times.iter() { + println!("{:<20} : {:?}", locator.0, locator.1); + } + println!(); } - println!(); println!("Breakdown for finding Environments:"); println!("-----------------------------------"); @@ -119,9 +192,78 @@ pub fn find_and_report_envs_stdio( println!() } } +} + +fn find_env( + executable: &PathBuf, + locators: &Arc>>, + environment: &dyn Environment, +) { + let collect_reporter = Arc::new(collect::create_reporter()); + let reporter = CacheReporter::new(collect_reporter.clone()); + let stdio_reporter = Arc::new(stdio::create_reporter(true)); + + let global_env_search_paths: Vec = get_search_paths_from_env_variables(environment); + + identify_python_executables_using_locators( + vec![executable.clone()], + locators, + &reporter, + &global_env_search_paths, + ); + + // Find the environment for the file provided. + let environments = collect_reporter.environments.lock().unwrap(); + if let Some(env) = environments + .iter() + .find(|e| e.symlinks.clone().unwrap_or_default().contains(executable)) + { + if let Some(manager) = &env.manager { + stdio_reporter.report_manager(manager); + } + stdio_reporter.report_environment(env); + } else { + warn!("Failed to find the environment for {:?}", executable); + } +} + +pub fn resolve_report_stdio(executable: PathBuf, verbose: bool) { + stdio::initialize_logger(if verbose { + log::LevelFilter::Trace + } else { + log::LevelFilter::Warn + }); + let now = SystemTime::now(); + let stdio_reporter = Arc::new(stdio::create_reporter(true)); + let reporter = CacheReporter::new(stdio_reporter.clone()); + let environment = EnvironmentApi::new(); + let conda_locator = Arc::new(Conda::from(&environment)); + let poetry_locator = Arc::new(Poetry::from(&environment)); + + let mut config = Configuration::default(); + if let Ok(cwd) = env::current_dir() { + config.workspace_directories = Some(vec![cwd]); + } + + let locators = create_locators(conda_locator.clone(), poetry_locator.clone(), &environment); + for locator in locators.iter() { + locator.configure(&config); + } + + if let Some(result) = resolve_environment(&executable, &locators, &environment) { + // + println!("Environment found for {:?}", executable); + let env = &result.resolved.unwrap_or(result.discovered); + if let Some(manager) = &env.manager { + reporter.report_manager(manager); + } + reporter.report_environment(env); + } else { + println!("No environment found for {:?}", executable); + } println!( - "Refresh completed in {}ms", + "Resolve completed in {}ms", now.elapsed().unwrap().as_millis() ) } diff --git a/crates/pet/src/main.rs b/crates/pet/src/main.rs index 0d6df826..26f473ca 100644 --- a/crates/pet/src/main.rs +++ b/crates/pet/src/main.rs @@ -1,9 +1,11 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +use std::path::PathBuf; + use clap::{Parser, Subcommand}; use jsonrpc::start_jsonrpc_server; -use pet::find_and_report_envs_stdio; +use pet::{find_and_report_envs_stdio, resolve_report_stdio, FindOptions}; mod find; mod jsonrpc; @@ -20,16 +22,42 @@ pub struct Cli { enum Commands { /// Finds the environments and reports them to the standard output. Find { + /// List of folders to search for environments. + /// The current directory is automatically used as a workspace folder if none provided. + #[arg(value_name = "WORKSPACE FOLDERS")] + workspace_dirs: Option>, + + /// List the environments found. #[arg(short, long)] list: bool, - /// Whether to display verbose output (defaults to just info). + /// Display verbose output (defaults to warnings). #[arg(short, long)] verbose: bool, - /// Whether to look for missing environments and report them (e.g. spawn conda and find what was missed). + /// Look for missing environments and report them (e.g. spawn conda and find what was missed). #[arg(short, long)] report_missing: bool, + + /// Exclusively search just the workspace directories. + /// I.e. exclude all global environments. + #[arg(short, long, conflicts_with = "global_only")] + workspace_only: bool, + + /// Exclusively search just the global environments (Conda, Poetry, Registry, etc). + /// I.e. do not search the workspace directories. + #[arg(short, long, conflicts_with = "workspace_only")] + global_only: bool, + }, + /// Resolves & reports the details of the the environment to the standard output. + Resolve { + /// Fully qualified path to the Python executable + #[arg(value_name = "PYTHON EXE")] + executable: PathBuf, + + /// Whether to display verbose output (defaults to warnings). + #[arg(short, long)] + verbose: bool, }, /// Starts the JSON RPC Server. Server, @@ -42,12 +70,30 @@ fn main() { list: true, verbose: false, report_missing: false, + workspace_dirs: None, + workspace_only: false, + global_only: false, }) { Commands::Find { list, verbose, report_missing, - } => find_and_report_envs_stdio(list, true, verbose, report_missing), + workspace_dirs, + workspace_only, + global_only, + } => find_and_report_envs_stdio(FindOptions { + print_list: list, + print_summary: true, + verbose, + report_missing, + workspace_dirs, + workspace_only, + global_only, + }), + Commands::Resolve { + executable, + verbose, + } => resolve_report_stdio(executable, verbose), Commands::Server => start_jsonrpc_server(), } } diff --git a/crates/pet/tests/ci_homebrew_container.rs b/crates/pet/tests/ci_homebrew_container.rs index 0be80a1b..8eaaa7c3 100644 --- a/crates/pet/tests/ci_homebrew_container.rs +++ b/crates/pet/tests/ci_homebrew_container.rs @@ -27,6 +27,7 @@ fn verify_python_in_homebrew_contaner() { Default::default(), &create_locators(conda_locator.clone(), poetry_locator.clone(), &environment), &environment, + None, ); let result = reporter.get_result(); diff --git a/crates/pet/tests/ci_jupyter_container.rs b/crates/pet/tests/ci_jupyter_container.rs index 6a649634..41428dee 100644 --- a/crates/pet/tests/ci_jupyter_container.rs +++ b/crates/pet/tests/ci_jupyter_container.rs @@ -45,6 +45,7 @@ fn verify_python_in_jupyter_contaner() { Default::default(), &create_locators(conda_locator.clone(), poetry_locator.clone(), &environment), &environment, + None, ); let result = reporter.get_result(); diff --git a/crates/pet/tests/ci_poetry.rs b/crates/pet/tests/ci_poetry.rs index 04f2da08..caa62ed7 100644 --- a/crates/pet/tests/ci_poetry.rs +++ b/crates/pet/tests/ci_poetry.rs @@ -32,7 +32,7 @@ fn verify_ci_poetry_global() { locator.configure(&config); } - find_and_report_envs(&reporter, Default::default(), &locators, &environment); + find_and_report_envs(&reporter, Default::default(), &locators, &environment, None); let result = reporter.get_result(); @@ -94,7 +94,7 @@ fn verify_ci_poetry_project() { locator.configure(&config); } - find_and_report_envs(&reporter, Default::default(), &locators, &environment); + find_and_report_envs(&reporter, Default::default(), &locators, &environment, None); let result = reporter.get_result(); diff --git a/crates/pet/tests/ci_test.rs b/crates/pet/tests/ci_test.rs index 51c0a174..a2d8c366 100644 --- a/crates/pet/tests/ci_test.rs +++ b/crates/pet/tests/ci_test.rs @@ -6,7 +6,10 @@ use std::{path::PathBuf, sync::Once}; use common::{does_version_match, resolve_test_path}; use lazy_static::lazy_static; use log::{error, trace}; -use pet::{locators::identify_python_environment_using_locators, resolve::resolve_environment}; +use pet::{ + find::identify_python_executables_using_locators, + locators::identify_python_environment_using_locators, resolve::resolve_environment, +}; use pet_core::{ arch::Architecture, python_environment::{PythonEnvironment, PythonEnvironmentKind}, @@ -14,6 +17,7 @@ use pet_core::{ use pet_env_var_path::get_search_paths_from_env_variables; use pet_poetry::Poetry; use pet_python_utils::env::PythonEnv; +use pet_reporter::{cache::CacheReporter, collect}; use regex::Regex; use serde::Deserialize; @@ -80,7 +84,7 @@ fn verify_validity_of_discovered_envs() { } // Find all environments on this machine. - find_and_report_envs(&reporter, Default::default(), &locators, &environment); + find_and_report_envs(&reporter, Default::default(), &locators, &environment, None); let result = reporter.get_result(); let environments = result.environments; @@ -110,6 +114,9 @@ fn verify_validity_of_discovered_envs() { // Verification 4 & 5: // Similarly for each environment use resolve method and verify we get the exact same information. verify_we_can_get_same_env_info_using_resolve_with_exe(exe, environment.clone()); + // Verification 6: + // Given the exe, verify we can use the `find` method in JSON RPC to get the details, without spawning Python. + verify_we_can_get_same_env_info_using_find_with_exe(exe, environment.clone()); } })); } @@ -183,6 +190,7 @@ fn check_if_pipenv_exists() { Default::default(), &create_locators(conda_locator.clone(), poetry_locator.clone(), &environment), &environment, + None, ); let result = reporter.get_result(); @@ -221,6 +229,7 @@ fn check_if_pyenv_virtualenv_exists() { Default::default(), &create_locators(conda_locator.clone(), poetry_locator.clone(), &environment), &environment, + None, ); let result = reporter.get_result(); @@ -375,6 +384,68 @@ fn verify_we_can_get_same_env_info_using_from_with_exe( ); } +fn verify_we_can_get_same_env_info_using_find_with_exe( + executable: &PathBuf, + environment: PythonEnvironment, +) { + // Assume we were given a path to the exe, then we use the `locator.try_from` method. + // We should be able to get the exct same information back given only the exe. + // + // Note: We will not not use the old locator objects, as we do not want any cached information. + // Hence create the locators all over again. + use pet::locators::create_locators; + use pet_conda::Conda; + use pet_core::{os_environment::EnvironmentApi, Configuration}; + use std::{env, sync::Arc}; + + let workspace_dir = PathBuf::from(env::var("GITHUB_WORKSPACE").unwrap_or_default()); + let os_environment = EnvironmentApi::new(); + let conda_locator = Arc::new(Conda::from(&os_environment)); + let poetry_locator = Arc::new(Poetry::from(&os_environment)); + let mut config = Configuration::default(); + let search_paths = vec![workspace_dir.clone()]; + config.workspace_directories = Some(search_paths.clone()); + let locators = create_locators( + conda_locator.clone(), + poetry_locator.clone(), + &os_environment, + ); + for locator in locators.iter() { + locator.configure(&config); + } + let global_env_search_paths: Vec = + get_search_paths_from_env_variables(&os_environment); + + let collect_reporter = Arc::new(collect::create_reporter()); + let reporter = CacheReporter::new(collect_reporter.clone()); + identify_python_executables_using_locators( + vec![executable.clone()], + &locators, + &reporter, + &global_env_search_paths, + ); + + let envs = collect_reporter.environments.lock().unwrap().clone(); + if envs.is_empty() { + panic!( + "Failed to find Python environment {:?}, details => {:?}", + executable, environment + ); + } + trace!( + "For exe {:?} we got Environment = {:?}, To compare against {:?}", + executable, + envs[0], + environment + ); + + compare_environments( + envs[0].clone(), + environment, + format!("find using exe {executable:?}").as_str(), + ); +} + fn compare_environments(actual: PythonEnvironment, expected: PythonEnvironment, method: &str) { let mut actual = actual.clone(); let mut expected = expected.clone(); @@ -588,6 +659,7 @@ fn verify_bin_usr_bin_user_local_are_separate_python_envs() { Default::default(), &create_locators(conda_locator.clone(), poetry_locator.clone(), &environment), &environment, + None, ); let result = reporter.get_result(); diff --git a/docs/JSONRPC.md b/docs/JSONRPC.md index 8b5565c2..7534f307 100644 --- a/docs/JSONRPC.md +++ b/docs/JSONRPC.md @@ -46,21 +46,21 @@ interface RefreshParams { * * Useful for VS Code so users can configure where they store virtual environments. */ - environment_directories: getCustomVirtualEnvDirs(), + environment_directories: string[]; /** * This is the path to the conda executable. * If conda is installed in the usual location, there's no need to update this value. * * Useful for VS Code so users can configure where they have installed Conda. */ - conda_executable: getPythonSettingAndUntildify(CONDAPATH_SETTING_KEY), + conda_executable?: string, /** * This is the path to the conda executable. * If Poetry is installed in the usual location, there's no need to update this value. * * Useful for VS Code so users can configure where they have installed Poetry. */ - poetry_executable: getPythonSettingAndUntildify('poetryPath'), + poetry_executable?: string, } interface RefreshResult { @@ -120,28 +120,24 @@ interface Environment { executable?: string; /** * The kind of the environment. - * - * If an environment is discovered and the kind is not know, then `Unknown` is used. - * I.e this is never Optional. */ - kind: + kind?: | "Conda" // Conda environment - | "Homebrew" // Homebrew installed Python - | "Pyenv" // Pyenv installed Python | "GlobalPaths" // Unknown Pyton environment, found in the PATH environment variable - | "PyenvVirtualEnv" // pyenv-virtualenv environment - | "Pipenv" // Pipenv environment - | "Poetry" // Poetry environment - | "MacPythonOrg" // Python installed from python.org on Mac - | "MacCommandLineTools" // Python installed from the Mac command line tools + | "Homebrew" // Homebrew installed Python | "LinuxGlobal" // Python installed from the system package manager on Linux + | "MacCommandLineTools" // Python installed from the Mac command line tools + | "MacPythonOrg" // Python installed from python.org on Mac | "MacXCode" // Python installed from XCode on Mac - | "Unknown" // Unknown Python environment + | "Pipenv" // Pipenv environment + | "Poetry" // Poetry environment + | "Pyenv" // Pyenv installed Python + | "PyenvVirtualEnv" // pyenv-virtualenv environment | "Venv" // Python venv environment (generally created using the `venv` module) | "VirtualEnv" // Python virtual environment | "VirtualEnvWrapper" // Virtualenvwrapper Environment - | "WindowsStore" // Python installed from the Windows Store - | "WindowsRegistry"; // Python installed & found in Windows Registry + | "WindowsRegistry" // Python installed & found in Windows Registry + | "WindowsStore"; // Python installed from the Windows Store /** * The version of the python executable. * This will at a minimum contain the 3 parts of the version such as `3.8.1`. From 5985017332632cc957028be4a67233b4dee46488 Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Thu, 11 Jul 2024 13:03:16 +1000 Subject: [PATCH 2/3] unwanted change --- crates/pet-python-utils/src/executable.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/pet-python-utils/src/executable.rs b/crates/pet-python-utils/src/executable.rs index 569fa44c..49c88772 100644 --- a/crates/pet-python-utils/src/executable.rs +++ b/crates/pet-python-utils/src/executable.rs @@ -92,7 +92,7 @@ pub fn find_executables>(env_path: T) -> Vec { python_executables } -pub fn is_python_executable_name(exe: &Path) -> bool { +fn is_python_executable_name(exe: &Path) -> bool { let name = exe .file_name() .unwrap_or_default() From f42da705ac9d16777bb56b57ec6b01c190048688 Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Thu, 11 Jul 2024 13:04:25 +1000 Subject: [PATCH 3/3] fixes --- crates/pet/tests/ci_test.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/pet/tests/ci_test.rs b/crates/pet/tests/ci_test.rs index a2d8c366..154dc464 100644 --- a/crates/pet/tests/ci_test.rs +++ b/crates/pet/tests/ci_test.rs @@ -148,6 +148,7 @@ fn check_if_virtualenvwrapper_exists() { Default::default(), &create_locators(conda_locator.clone(), poetry_locator.clone(), &environment), &environment, + None, ); let result = reporter.get_result();