diff --git a/Cargo.lock b/Cargo.lock index 33cab1ff..cce478a0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -210,6 +210,7 @@ dependencies = [ "pet-pyenv", "pet-python-utils", "pet-reporter", + "pet-telemetry", "pet-venv", "pet-virtualenv", "pet-virtualenvwrapper", @@ -223,6 +224,14 @@ dependencies = [ [[package]] name = "pet-cache" version = "0.1.0" +dependencies = [ + "log", + "pet-core", + "pet-fs", + "pet-python-utils", + "serde", + "serde_json", +] [[package]] name = "pet-conda" @@ -269,6 +278,9 @@ dependencies = [ [[package]] name = "pet-fs" version = "0.1.0" +dependencies = [ + "log", +] [[package]] name = "pet-global" @@ -288,10 +300,6 @@ dependencies = [ "pet-virtualenv", ] -[[package]] -name = "pet-hatch" -version = "0.1.0" - [[package]] name = "pet-homebrew" version = "0.1.0" @@ -402,6 +410,19 @@ dependencies = [ "serde_json", ] +[[package]] +name = "pet-telemetry" +version = "0.1.0" +dependencies = [ + "env_logger", + "lazy_static", + "log", + "pet-core", + "pet-fs", + "pet-python-utils", + "regex", +] + [[package]] name = "pet-venv" version = "0.1.0" diff --git a/crates/pet-cache/Cargo.toml b/crates/pet-cache/Cargo.toml index b6768deb..192f7517 100644 --- a/crates/pet-cache/Cargo.toml +++ b/crates/pet-cache/Cargo.toml @@ -4,3 +4,9 @@ version = "0.1.0" edition = "2021" [dependencies] +pet-fs = { path = "../pet-fs" } +pet-python-utils = { path = "../pet-python-utils" } +serde = { version = "1.0.152", features = ["derive"] } +serde_json = "1.0.93" +pet-core = { path = "../pet-core" } +log = "0.4.21" diff --git a/crates/pet-cache/src/lib.rs b/crates/pet-cache/src/lib.rs index 7d12d9af..95279817 100644 --- a/crates/pet-cache/src/lib.rs +++ b/crates/pet-cache/src/lib.rs @@ -1,14 +1,9 @@ -pub fn add(left: usize, right: usize) -> usize { - left + right -} +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. -#[cfg(test)] -mod tests { - use super::*; +use pet_core::python_environment::PythonEnvironment; - #[test] - fn it_works() { - let result = add(2, 2); - assert_eq!(result, 4); - } +pub trait Cache { + fn get>(&self, executable: P) -> Option; + fn set>(&self, environment: PythonEnvironment); } diff --git a/crates/pet-conda/src/lib.rs b/crates/pet-conda/src/lib.rs index 64b2e813..12a54cd2 100644 --- a/crates/pet-conda/src/lib.rs +++ b/crates/pet-conda/src/lib.rs @@ -104,9 +104,6 @@ impl Conda { } impl Locator for Conda { - fn resolve(&self, _env: &PythonEnvironment) -> Option { - todo!() - } fn from(&self, env: &PythonEnv) -> Option { if let Some(ref path) = env.prefix { let mut environments = self.environments.lock().unwrap(); @@ -148,6 +145,11 @@ impl Locator for Conda { } fn find(&self, reporter: &dyn Reporter) { + // if we're calling this again, then clear what ever cache we have. + let mut environments = self.environments.lock().unwrap(); + environments.clear(); + drop(environments); + let env_vars = self.env_vars.clone(); thread::scope(|s| { // 1. Get a list of all know conda environments file paths diff --git a/crates/pet-core/src/arch.rs b/crates/pet-core/src/arch.rs index 177d906d..caaae54c 100644 --- a/crates/pet-core/src/arch.rs +++ b/crates/pet-core/src/arch.rs @@ -20,3 +20,19 @@ impl PartialOrd for Architecture { Some(self.cmp(other)) } } + +impl std::fmt::Display for Architecture { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!( + f, + "{}", + if *self == Architecture::X64 { + "x64" + } else { + "x86" + } + ) + .unwrap_or_default(); + Ok(()) + } +} diff --git a/crates/pet-core/src/cache.rs b/crates/pet-core/src/cache.rs new file mode 100644 index 00000000..92b9f1de --- /dev/null +++ b/crates/pet-core/src/cache.rs @@ -0,0 +1,7 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +pub trait Cache { + fn get>(&self, executable: P) -> Option; + fn set>(&self, environment: PythonEnvironment); +} diff --git a/crates/pet-core/src/lib.rs b/crates/pet-core/src/lib.rs index 0065232f..2ff0778a 100644 --- a/crates/pet-core/src/lib.rs +++ b/crates/pet-core/src/lib.rs @@ -11,7 +11,7 @@ pub mod manager; pub mod os_environment; pub mod python_environment; pub mod reporter; -// pub mod telemetry; +pub mod telemetry; #[derive(Debug, Clone)] pub struct LocatorResult { @@ -28,16 +28,6 @@ pub trait Locator: Send + Sync { * This is because the `from` will do a best effort to get the environment information without spawning Python. */ fn from(&self, env: &PythonEnv) -> Option; - /** - * Given a Python environment, get all of the information by spawning the Python executable. - * E.g. version, sysprefix, etc ... - * - * I.e. use this to test whether an environment is of a specific type. - */ - fn resolve(&self, env: &PythonEnvironment) -> Option { - // TODO: Implement this. - Some(env.clone()) - } /** * Finds all environments specific to this locator. */ diff --git a/crates/pet-core/src/manager.rs b/crates/pet-core/src/manager.rs index 434403d3..934ab9d1 100644 --- a/crates/pet-core/src/manager.rs +++ b/crates/pet-core/src/manager.rs @@ -60,3 +60,19 @@ impl EnvManager { } } } + +impl std::fmt::Display for EnvManager { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + writeln!(f, "Manager ({:?})", self.tool).unwrap_or_default(); + writeln!( + f, + " Executable : {}", + self.executable.to_str().unwrap_or_default() + ) + .unwrap_or_default(); + if let Some(version) = &self.version { + writeln!(f, " Version : {}", version).unwrap_or_default(); + } + Ok(()) + } +} diff --git a/crates/pet-core/src/python_environment.rs b/crates/pet-core/src/python_environment.rs index 7611fc81..9141cc47 100644 --- a/crates/pet-core/src/python_environment.rs +++ b/crates/pet-core/src/python_environment.rs @@ -2,6 +2,7 @@ // Licensed under the MIT License. use pet_fs::path::norm_case; +use pet_python_utils::executable::get_shortest_executable; use serde::{Deserialize, Serialize}; use std::path::PathBuf; @@ -67,6 +68,7 @@ pub struct PythonEnvironment { // E.g. in the case of Homebrew there are a number of symlinks that are created. pub symlinks: Option>, } + impl Ord for PythonEnvironment { fn cmp(&self, other: &Self) -> std::cmp::Ordering { format!( @@ -126,6 +128,68 @@ impl PythonEnvironment { } } +impl std::fmt::Display for PythonEnvironment { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + writeln!(f, "Environment ({:?})", self.category).unwrap_or_default(); + if let Some(name) = &self.display_name { + writeln!(f, " Display-Name: {}", name).unwrap_or_default(); + } + if let Some(name) = &self.name { + writeln!(f, " Name : {}", name).unwrap_or_default(); + } + if let Some(exe) = &self.executable { + writeln!(f, " Executable : {}", exe.to_str().unwrap_or_default()) + .unwrap_or_default(); + } + if let Some(version) = &self.version { + writeln!(f, " Version : {}", version).unwrap_or_default(); + } + if let Some(prefix) = &self.prefix { + writeln!( + f, + " Prefix : {}", + prefix.to_str().unwrap_or_default() + ) + .unwrap_or_default(); + } + if let Some(project) = &self.project { + writeln!(f, " Project : {}", project.to_str().unwrap()).unwrap_or_default(); + } + if let Some(arch) = &self.arch { + writeln!(f, " Architecture: {}", arch).unwrap_or_default(); + } + if let Some(manager) = &self.manager { + writeln!( + f, + " Manager : {:?}, {}", + manager.tool, + manager.executable.to_str().unwrap_or_default() + ) + .unwrap_or_default(); + } + if let Some(symlinks) = &self.symlinks { + let mut symlinks = symlinks.clone(); + symlinks.sort_by(|a, b| { + a.to_str() + .unwrap_or_default() + .len() + .cmp(&b.to_str().unwrap_or_default().len()) + }); + + if !symlinks.is_empty() { + for (i, symlink) in symlinks.iter().enumerate() { + if i == 0 { + writeln!(f, " Symlinks : {:?}", symlink).unwrap_or_default(); + } else { + writeln!(f, " : {:?}", symlink).unwrap_or_default(); + } + } + } + } + Ok(()) + } +} + #[derive(Serialize, Deserialize, Clone)] #[serde(rename_all = "camelCase")] #[derive(Debug)] @@ -177,7 +241,7 @@ impl PythonEnvironmentBuilder { } } } - self.update_symlinks(self.symlinks.clone()); + self.update_symlinks_and_exe(self.symlinks.clone()); self } @@ -213,11 +277,11 @@ impl PythonEnvironmentBuilder { } pub fn symlinks(mut self, symlinks: Option>) -> Self { - self.update_symlinks(symlinks); + self.update_symlinks_and_exe(symlinks); self } - fn update_symlinks(&mut self, symlinks: Option>) { + fn update_symlinks_and_exe(&mut self, symlinks: Option>) { let mut all = vec![]; if let Some(ref exe) = self.executable { all.push(exe.clone()); @@ -228,7 +292,15 @@ impl PythonEnvironmentBuilder { all.sort(); all.dedup(); - self.symlinks = if all.is_empty() { None } else { Some(all) }; + self.symlinks = if all.is_empty() { + None + } else { + Some(all.clone()) + }; + if let Some(executable) = &self.executable { + self.executable = + Some(get_shortest_executable(&Some(all.clone())).unwrap_or(executable.clone())); + } } pub fn build(self) -> PythonEnvironment { diff --git a/crates/pet-core/src/telemetry/inaccurate_python_info.rs b/crates/pet-core/src/telemetry/inaccurate_python_info.rs new file mode 100644 index 00000000..50034b27 --- /dev/null +++ b/crates/pet-core/src/telemetry/inaccurate_python_info.rs @@ -0,0 +1,44 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use crate::python_environment::PythonEnvironmentCategory; + +/// Information about an environment that was discovered to be inaccurate. +/// If the discovered information is None, then it means that the information was not found. +/// And we will not report that as an inaccuracy. +pub struct InaccuratePythonEnvironmentInfo { + /// Python Env category + pub category: PythonEnvironmentCategory, + /// Whether the actual exe is not what we expected. + pub invalid_executable: Option, + /// Whether the actual exe was not even in the list of symlinks that we expected. + pub executable_not_in_symlinks: Option, + /// Whether the prefix is not what we expected. + pub invalid_prefix: Option, + /// Whether the version is not what we expected. + pub invalid_version: Option, + /// Whether the architecture is not what we expected. + pub invalid_arch: Option, +} + +impl std::fmt::Display for InaccuratePythonEnvironmentInfo { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + writeln!(f, "Environment {:?} incorrectly identified", self.category).unwrap_or_default(); + if self.invalid_executable.unwrap_or_default() { + writeln!(f, " Executable is incorrect").unwrap_or_default(); + } + if self.executable_not_in_symlinks.unwrap_or_default() { + writeln!(f, " Executable is not in the list of symlinks").unwrap_or_default(); + } + if self.invalid_prefix.unwrap_or_default() { + writeln!(f, " Prefix is incorrect").unwrap_or_default(); + } + if self.invalid_version.unwrap_or_default() { + writeln!(f, " Version is incorrect").unwrap_or_default(); + } + if self.invalid_arch.unwrap_or_default() { + writeln!(f, " Architecture is incorrect").unwrap_or_default(); + } + Ok(()) + } +} diff --git a/crates/pet-core/src/telemetry/mod.rs b/crates/pet-core/src/telemetry/mod.rs new file mode 100644 index 00000000..32dd26a3 --- /dev/null +++ b/crates/pet-core/src/telemetry/mod.rs @@ -0,0 +1,25 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use inaccurate_python_info::InaccuratePythonEnvironmentInfo; + +pub mod inaccurate_python_info; + +pub type NumberOfCustomSearchPaths = u32; + +pub enum TelemetryEvent { + /// Total time taken to search for Global environments. + GlobalEnvironmentsSearchCompleted(std::time::Duration), + /// Total time taken to search for Global Virtual environments. + GlobalVirtualEnvironmentsSearchCompleted(std::time::Duration), + /// Total time taken to search for environments in the PATH environment variable. + GlobalPathVariableEnvironmentsSearchCompleted(std::time::Duration), + /// Total time taken to search for environments in specific paths provided by the user. + /// This generally maps to workspace folders in Python extension. + AllSearchPathsEnvironmentsSearchCompleted(std::time::Duration, NumberOfCustomSearchPaths), + /// Total time taken to search for all environments in all locations. + /// This is the max of all of the other `SearchCompleted` durations. + SearchCompleted(std::time::Duration), + /// Sent when an the information for an environment discovered is not accurate. + InaccuratePythonEnvironmentInfo(InaccuratePythonEnvironmentInfo), +} diff --git a/crates/pet-fs/Cargo.toml b/crates/pet-fs/Cargo.toml index 72834602..2d2e9a56 100644 --- a/crates/pet-fs/Cargo.toml +++ b/crates/pet-fs/Cargo.toml @@ -4,3 +4,4 @@ version = "0.1.0" edition = "2021" [dependencies] +log = "0.4.21" diff --git a/crates/pet-hatch/Cargo.toml b/crates/pet-hatch/Cargo.toml deleted file mode 100644 index 01246f89..00000000 --- a/crates/pet-hatch/Cargo.toml +++ /dev/null @@ -1,6 +0,0 @@ -[package] -name = "pet-hatch" -version = "0.1.0" -edition = "2021" - -[dependencies] diff --git a/crates/pet-hatch/src/lib.rs b/crates/pet-hatch/src/lib.rs deleted file mode 100644 index 7d12d9af..00000000 --- a/crates/pet-hatch/src/lib.rs +++ /dev/null @@ -1,14 +0,0 @@ -pub fn add(left: usize, right: usize) -> usize { - left + right -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn it_works() { - let result = add(2, 2); - assert_eq!(result, 4); - } -} diff --git a/crates/pet-homebrew/src/environments.rs b/crates/pet-homebrew/src/environments.rs index 82dd64e6..063dc1c0 100644 --- a/crates/pet-homebrew/src/environments.rs +++ b/crates/pet-homebrew/src/environments.rs @@ -7,7 +7,6 @@ use pet_core::python_environment::{ PythonEnvironment, PythonEnvironmentBuilder, PythonEnvironmentCategory, }; use pet_fs::path::resolve_symlink; -use pet_python_utils::executable::get_shortest_executable; use regex::Regex; use std::path::{Path, PathBuf}; @@ -20,14 +19,14 @@ pub fn get_python_info( python_exe_from_bin_dir: &Path, resolved_exe: &Path, ) -> Option { - let user_friendly_exe = python_exe_from_bin_dir; + // let user_friendly_exe = python_exe_from_bin_dir; let python_version = resolved_exe.to_string_lossy().to_string(); let version = match PYTHON_VERSION.captures(&python_version) { Some(captures) => captures.get(1).map(|version| version.as_str().to_string()), None => None, }; - let mut symlinks = vec![user_friendly_exe.to_path_buf()]; + let mut symlinks = vec![python_exe_from_bin_dir.to_path_buf()]; if let Some(version) = &version { symlinks.append(&mut get_known_symlinks(resolved_exe, version)); } @@ -45,11 +44,9 @@ pub fn get_python_info( symlinks.sort(); symlinks.dedup(); - let user_friendly_exe = - get_shortest_executable(&Some(symlinks.clone())).unwrap_or(user_friendly_exe.to_path_buf()); let env = PythonEnvironmentBuilder::new(PythonEnvironmentCategory::Homebrew) - .executable(Some(user_friendly_exe.to_path_buf())) + .executable(Some(python_exe_from_bin_dir.to_path_buf())) .version(version) .prefix(get_prefix(resolved_exe)) .symlinks(Some(symlinks)) diff --git a/crates/pet-jsonrpc/src/core.rs b/crates/pet-jsonrpc/src/core.rs deleted file mode 100644 index dc9315f8..00000000 --- a/crates/pet-jsonrpc/src/core.rs +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -use std::io::{self, Write}; - -use serde::{Deserialize, Serialize}; - -#[derive(Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -#[derive(Debug)] -struct AnyMethodMessage { - pub jsonrpc: String, - pub method: &'static str, - pub params: Option, -} - -pub fn send_message(method: &'static str, params: Option) { - let payload = AnyMethodMessage { - jsonrpc: "2.0".to_string(), - method, - params, - }; - let message = serde_json::to_string(&payload).unwrap(); - print!( - "Content-Length: {}\r\nContent-Type: application/vscode-jsonrpc; charset=utf-8\r\n\r\n{}", - message.len(), - message - ); - let _ = io::stdout().flush(); -} -pub fn send_reply(id: u32, payload: Option) { - let payload = serde_json::json!({ - "jsonrpc": "2.0", - "result": payload, - "id": id - }); - let message = serde_json::to_string(&payload).unwrap(); - print!( - "Content-Length: {}\r\nContent-Type: application/vscode-jsonrpc; charset=utf-8\r\n\r\n{}", - message.len(), - message - ); - let _ = io::stdout().flush(); -} - -pub fn send_error(id: Option, code: i32, message: String) { - let payload = serde_json::json!({ - "jsonrpc": "2.0", - "error": { "code": code, "message": message }, - "id": id - }); - let message = serde_json::to_string(&payload).unwrap(); - print!( - "Content-Length: {}\r\nContent-Type: application/vscode-jsonrpc; charset=utf-8\r\n\r\n{}", - message.len(), - message - ); - let _ = io::stdout().flush(); -} diff --git a/crates/pet-jsonrpc/src/lib.rs b/crates/pet-jsonrpc/src/lib.rs index bc0376a4..16902ecf 100644 --- a/crates/pet-jsonrpc/src/lib.rs +++ b/crates/pet-jsonrpc/src/lib.rs @@ -1,17 +1,60 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -mod core; +use serde::{Deserialize, Serialize}; +use std::io::{self, Write}; + pub mod server; -pub fn send_message(method: &'static str, params: Option) { - core::send_message(method, params) +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[derive(Debug)] +struct AnyMethodMessage { + pub jsonrpc: String, + pub method: &'static str, + pub params: Option, } +pub fn send_message(method: &'static str, params: Option) { + let payload = AnyMethodMessage { + jsonrpc: "2.0".to_string(), + method, + params, + }; + let message = serde_json::to_string(&payload).unwrap(); + print!( + "Content-Length: {}\r\nContent-Type: application/vscode-jsonrpc; charset=utf-8\r\n\r\n{}", + message.len(), + message + ); + let _ = io::stdout().flush(); +} pub fn send_reply(id: u32, payload: Option) { - core::send_reply(id, payload) + let payload = serde_json::json!({ + "jsonrpc": "2.0", + "result": payload, + "id": id + }); + let message = serde_json::to_string(&payload).unwrap(); + print!( + "Content-Length: {}\r\nContent-Type: application/vscode-jsonrpc; charset=utf-8\r\n\r\n{}", + message.len(), + message + ); + let _ = io::stdout().flush(); } pub fn send_error(id: Option, code: i32, message: String) { - core::send_error(id, code, message) + let payload = serde_json::json!({ + "jsonrpc": "2.0", + "error": { "code": code, "message": message }, + "id": id + }); + let message = serde_json::to_string(&payload).unwrap(); + print!( + "Content-Length: {}\r\nContent-Type: application/vscode-jsonrpc; charset=utf-8\r\n\r\n{}", + message.len(), + message + ); + let _ = io::stdout().flush(); } diff --git a/crates/pet-jsonrpc/src/server.rs b/crates/pet-jsonrpc/src/server.rs index 69615e2b..dfe7c09a 100644 --- a/crates/pet-jsonrpc/src/server.rs +++ b/crates/pet-jsonrpc/src/server.rs @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -use crate::core::send_error; +use crate::send_error; use serde_json::{self, Value}; use std::{ collections::HashMap, diff --git a/crates/pet-linux-global-python/src/lib.rs b/crates/pet-linux-global-python/src/lib.rs index f7f3746b..c75ce4e6 100644 --- a/crates/pet-linux-global-python/src/lib.rs +++ b/crates/pet-linux-global-python/src/lib.rs @@ -7,7 +7,6 @@ use pet_core::{ Locator, }; use pet_fs::path::resolve_symlink; -use pet_python_utils::executable::get_shortest_executable; use pet_python_utils::version; use pet_python_utils::{ env::{PythonEnv, ResolvedPythonEnv}, @@ -116,12 +115,10 @@ impl Locator for LinuxGlobalPython { prefix = Some(resolved_env.prefix); } } - let user_friendly_exe = - get_shortest_executable(&Some(symlinks.clone())).unwrap_or(env.executable.clone()); Some( PythonEnvironmentBuilder::new(PythonEnvironmentCategory::MacCommandLineTools) - .executable(Some(user_friendly_exe)) + .executable(Some(env.executable.clone())) .version(version) .prefix(prefix) .symlinks(Some(symlinks)) diff --git a/crates/pet-mac-commandlinetools/src/lib.rs b/crates/pet-mac-commandlinetools/src/lib.rs index 82306593..c0b36f6c 100644 --- a/crates/pet-mac-commandlinetools/src/lib.rs +++ b/crates/pet-mac-commandlinetools/src/lib.rs @@ -7,7 +7,6 @@ use pet_core::{ Locator, }; use pet_fs::path::resolve_symlink; -use pet_python_utils::executable::get_shortest_executable; use pet_python_utils::version; use pet_python_utils::{ env::{PythonEnv, ResolvedPythonEnv}, @@ -116,12 +115,10 @@ impl Locator for MacCmdLineTools { prefix = Some(resolved_env.prefix); } } - let user_friendly_exe = - get_shortest_executable(&Some(symlinks.clone())).unwrap_or(env.executable.clone()); Some( PythonEnvironmentBuilder::new(PythonEnvironmentCategory::MacCommandLineTools) - .executable(Some(user_friendly_exe)) + .executable(Some(env.executable.clone())) .version(version) .prefix(prefix) .symlinks(Some(symlinks)) diff --git a/crates/pet-mac-python-org/src/lib.rs b/crates/pet-mac-python-org/src/lib.rs index 7bddc870..d9f36a8b 100644 --- a/crates/pet-mac-python-org/src/lib.rs +++ b/crates/pet-mac-python-org/src/lib.rs @@ -8,7 +8,7 @@ use pet_core::{ }; use pet_fs::path::resolve_symlink; use pet_python_utils::env::PythonEnv; -use pet_python_utils::executable::{find_executables, get_shortest_executable}; +use pet_python_utils::executable::find_executables; use pet_python_utils::version; use std::fs; use std::path::PathBuf; @@ -111,15 +111,12 @@ impl Locator for MacPythonOrg { } } - let user_friendly_exe = - get_shortest_executable(&Some(symlinks.clone())).unwrap_or(executable.clone()); - symlinks.sort(); symlinks.dedup(); Some( PythonEnvironmentBuilder::new(PythonEnvironmentCategory::MacPythonOrg) - .executable(Some(user_friendly_exe)) + .executable(Some(executable.clone())) .version(Some(version)) .prefix(Some(prefix.to_path_buf())) .symlinks(Some(symlinks)) diff --git a/crates/pet-pipenv/src/lib.rs b/crates/pet-pipenv/src/lib.rs index 9e6ef9f6..703d7672 100644 --- a/crates/pet-pipenv/src/lib.rs +++ b/crates/pet-pipenv/src/lib.rs @@ -10,6 +10,7 @@ use pet_core::{ }; use pet_fs::path::norm_case; use pet_python_utils::env::PythonEnv; +use pet_python_utils::executable::find_executables; use pet_python_utils::version; use std::path::Path; use std::{fs, path::PathBuf}; @@ -87,10 +88,12 @@ impl Locator for PipEnv { } } } + let bin = env.executable.parent()?; + let symlinks = find_executables(bin); let mut version = env.version.clone(); if version.is_none() && prefix.is_some() { if let Some(prefix) = &prefix { - version = version::from_prefix(prefix); + version = version::from_creator_for_virtual_env(prefix); } } Some( @@ -99,6 +102,7 @@ impl Locator for PipEnv { .version(version) .prefix(prefix) .project(Some(project_path)) + .symlinks(Some(symlinks)) .build(), ) } diff --git a/crates/pet-python-utils/src/env.rs b/crates/pet-python-utils/src/env.rs index 61a7d958..88b5d9eb 100644 --- a/crates/pet-python-utils/src/env.rs +++ b/crates/pet-python-utils/src/env.rs @@ -6,6 +6,8 @@ use pet_fs::path::norm_case; use serde::Deserialize; use std::path::{Path, PathBuf}; +use crate::pyvenv_cfg::PyVenvCfg; + #[derive(Debug)] pub struct PythonEnv { /// Executable of the Python environment. @@ -33,6 +35,19 @@ impl PythonEnv { if let Some(value) = prefix { prefix = norm_case(value).into(); } + // if the prefix is not defined, try to get this. + // For instance, if the file is bin/python or Scripts/python + // And we have a pyvenv.cfg file in the parent directory, then we can get the prefix. + if prefix.is_none() { + let mut exe = executable.clone(); + exe.pop(); + if exe.ends_with("Scripts") || exe.ends_with("bin") { + exe.pop(); + if PyVenvCfg::find(&exe).is_some() { + prefix = Some(exe); + } + } + } Self { executable: norm_case(executable), prefix, diff --git a/crates/pet-python-utils/src/executable.rs b/crates/pet-python-utils/src/executable.rs index 7388cd56..eae51411 100644 --- a/crates/pet-python-utils/src/executable.rs +++ b/crates/pet-python-utils/src/executable.rs @@ -2,6 +2,7 @@ // Licensed under the MIT License. use lazy_static::lazy_static; +use log::trace; use regex::Regex; use std::{ fs, @@ -67,8 +68,7 @@ pub fn find_executables>(env_path: T) -> Vec { "python3" }; - // Unfortunately we must enumerate - // E.g. on linux /home/linuxbrew/.linuxbrew/bin does not contain a `python` file + // On linux /home/linuxbrew/.linuxbrew/bin does not contain a `python` file // If you install python@3.10, then only a python3.10 exe is created in that bin directory. // As a compromise, we only enumerate if this is a bin directory and there are no python exes // Else enumerating entire directories is very expensive. @@ -197,3 +197,39 @@ mod tests { )); } } + +pub fn should_search_for_environments_in_path>(path: &P) -> bool { + // Never search in the .git folder + // Never search in the node_modules folder + // Mostly copied from https://github.com/github/gitignore/blob/main/Python.gitignore + let folders_to_ignore = [ + "node_modules", + ".git", + ".tox", + ".nox", + ".hypothesis", + ".ipynb_checkpoints", + ".eggs", + ".coverage", + ".cache", + ".pyre", + ".ptype", + ".pytest_cache", + "__pycache__", + "__pypackages__", + ".mypy_cache", + "cython_debug", + "env.bak", + "venv.bak", + "Scripts", // If the folder ends bin/scripts, then ignore it, as the parent is most likely an env. + "bin", // If the folder ends bin/scripts, then ignore it, as the parent is most likely an env. + ]; + for folder in folders_to_ignore.iter() { + if path.as_ref().ends_with(folder) { + trace!("Ignoring folder: {:?}", path.as_ref()); + return false; + } + } + + true +} diff --git a/crates/pet-reporter/src/environment.rs b/crates/pet-reporter/src/environment.rs index f19c6089..b1b42b3e 100644 --- a/crates/pet-reporter/src/environment.rs +++ b/crates/pet-reporter/src/environment.rs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +use crate::manager::Manager; use log::error; use pet_core::{ arch::Architecture, @@ -9,8 +10,6 @@ use pet_core::{ use serde::{Deserialize, Serialize}; use std::path::PathBuf; -use crate::manager::Manager; - // We want to maintain full control over serialization instead of relying on the enums or the like. // Else its too easy to break the API by changing the enum variants. fn python_category_to_string(category: &PythonEnvironmentCategory) -> &'static str { @@ -49,84 +48,22 @@ pub struct Environment { pub display_name: Option, pub name: Option, pub executable: Option, - pub category: &'static str, + pub category: String, pub version: Option, pub prefix: Option, pub manager: Option, pub project: Option, - pub arch: Option<&'static str>, + pub arch: Option, pub symlinks: Option>, } -impl std::fmt::Display for Environment { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - writeln!(f, "Environment ({})", self.category).unwrap_or_default(); - if let Some(name) = &self.display_name { - writeln!(f, " Display-Name: {}", name).unwrap_or_default(); - } - if let Some(name) = &self.name { - writeln!(f, " Name : {}", name).unwrap_or_default(); - } - if let Some(exe) = &self.executable { - writeln!(f, " Executable : {}", exe.to_str().unwrap_or_default()) - .unwrap_or_default(); - } - if let Some(version) = &self.version { - writeln!(f, " Version : {}", version).unwrap_or_default(); - } - if let Some(prefix) = &self.prefix { - writeln!( - f, - " Prefix : {}", - prefix.to_str().unwrap_or_default() - ) - .unwrap_or_default(); - } - if let Some(project) = &self.project { - writeln!(f, " Project : {}", project.to_str().unwrap()).unwrap_or_default(); - } - if let Some(arch) = &self.arch { - writeln!(f, " Architecture: {}", arch).unwrap_or_default(); - } - if let Some(manager) = &self.manager { - writeln!( - f, - " Manager : {}, {}", - manager.tool, - manager.executable.to_str().unwrap_or_default() - ) - .unwrap_or_default(); - } - if let Some(symlinks) = &self.symlinks { - let mut symlinks = symlinks.clone(); - symlinks.sort_by(|a, b| { - a.to_str() - .unwrap_or_default() - .len() - .cmp(&b.to_str().unwrap_or_default().len()) - }); - - if !symlinks.is_empty() { - for (i, symlink) in symlinks.iter().enumerate() { - if i == 0 { - writeln!(f, " Symlinks : {:?}", symlink).unwrap_or_default(); - } else { - writeln!(f, " : {:?}", symlink).unwrap_or_default(); - } - } - } - } - Ok(()) - } -} - impl Environment { pub fn from(env: &PythonEnvironment) -> Environment { Environment { display_name: env.display_name.clone(), name: env.name.clone(), executable: env.executable.clone(), - category: python_category_to_string(&env.category), + category: python_category_to_string(&env.category).to_string(), version: env.version.clone(), prefix: env.prefix.clone(), manager: match &env.manager { @@ -134,7 +71,11 @@ impl Environment { None => None, }, project: env.project.clone(), - arch: env.arch.as_ref().map(architecture_to_string), + arch: env + .arch + .as_ref() + .map(architecture_to_string) + .map(|s| s.to_string()), symlinks: env.symlinks.clone(), } } diff --git a/crates/pet-reporter/src/lib.rs b/crates/pet-reporter/src/lib.rs index 0d17a80f..e7836d01 100644 --- a/crates/pet-reporter/src/lib.rs +++ b/crates/pet-reporter/src/lib.rs @@ -1,8 +1,8 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -mod environment; +pub mod environment; pub mod jsonrpc; -mod manager; +pub mod manager; pub mod stdio; pub mod test; diff --git a/crates/pet-reporter/src/manager.rs b/crates/pet-reporter/src/manager.rs index 29dc58e0..a46cfe41 100644 --- a/crates/pet-reporter/src/manager.rs +++ b/crates/pet-reporter/src/manager.rs @@ -18,7 +18,7 @@ fn tool_to_string(tool: &EnvManagerType) -> &'static str { pub struct Manager { pub executable: PathBuf, pub version: Option, - pub tool: &'static str, + pub tool: String, } impl Manager { @@ -26,23 +26,7 @@ impl Manager { Manager { executable: env.executable.clone(), version: env.version.clone(), - tool: tool_to_string(&env.tool), + tool: tool_to_string(&env.tool).to_string(), } } } - -impl std::fmt::Display for Manager { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - writeln!(f, "Manager ({})", self.tool).unwrap_or_default(); - writeln!( - f, - " Executable : {}", - self.executable.to_str().unwrap_or_default() - ) - .unwrap_or_default(); - if let Some(version) = &self.version { - writeln!(f, " Version : {}", version).unwrap_or_default(); - } - Ok(()) - } -} diff --git a/crates/pet-reporter/src/stdio.rs b/crates/pet-reporter/src/stdio.rs index fe3dc79f..670aa8bb 100644 --- a/crates/pet-reporter/src/stdio.rs +++ b/crates/pet-reporter/src/stdio.rs @@ -1,10 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -use crate::{ - environment::{get_environment_key, Environment}, - manager::Manager, -}; +use crate::environment::get_environment_key; use env_logger::Builder; use log::LevelFilter; use pet_core::{manager::EnvManager, python_environment::PythonEnvironment, reporter::Reporter}; @@ -26,7 +23,7 @@ impl Reporter for StdioReporter { if !reported_managers.contains(&manager.executable) { reported_managers.insert(manager.executable.clone()); let prefix = format!("{}.", reported_managers.len()); - println!("{:<3}{}", prefix, Manager::from(manager)) + println!("{:<3}{}", prefix, manager) } } @@ -36,7 +33,7 @@ impl Reporter for StdioReporter { if !reported_environments.contains(&key) { reported_environments.insert(key.clone()); let prefix = format!("{}.", reported_environments.len()); - println!("{:<3}{}", prefix, Environment::from(env)) + println!("{:<3}{}", prefix, env) } } } diff --git a/crates/pet-telemetry/Cargo.toml b/crates/pet-telemetry/Cargo.toml new file mode 100644 index 00000000..d42b1b98 --- /dev/null +++ b/crates/pet-telemetry/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "pet-telemetry" +version = "0.1.0" +edition = "2021" + +[dependencies] +pet-core = { path = "../pet-core" } +pet-fs = { path = "../pet-fs" } +pet-python-utils = { path = "../pet-python-utils" } +log = "0.4.21" +env_logger = "0.10.2" +lazy_static = "1.4.0" +regex = "1.10.4" diff --git a/crates/pet-telemetry/src/lib.rs b/crates/pet-telemetry/src/lib.rs new file mode 100644 index 00000000..955b0805 --- /dev/null +++ b/crates/pet-telemetry/src/lib.rs @@ -0,0 +1,86 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::ops::Deref; + +use lazy_static::lazy_static; +use log::warn; +use pet_core::{ + python_environment::PythonEnvironment, reporter::Reporter, + telemetry::inaccurate_python_info::InaccuratePythonEnvironmentInfo, +}; +use pet_fs::path::norm_case; +use regex::Regex; + +lazy_static! { + static ref PYTHON_VERSION: Regex = Regex::new(r"(\d+\.\d+\.\d+).*") + .expect("Error creating Python Version Regex for comparison"); +} + +pub fn report_inaccuracies_identified_after_resolving( + _reporter: &dyn Reporter, + env: &PythonEnvironment, + resolved: &PythonEnvironment, +) -> Option<()> { + let known_symlinks = env.symlinks.clone().unwrap_or_default(); + let resolved_executable = &resolved.executable.clone()?; + let norm_cased_executable = norm_case(resolved_executable); + + let mut invalid_executable = env.executable.clone().unwrap_or_default() + != resolved_executable.deref() + && env.executable.clone().unwrap_or_default() != norm_cased_executable; + if env.executable.clone().is_none() { + invalid_executable = false; + } + + let mut executable_not_in_symlinks = !known_symlinks.contains(resolved_executable) + && !known_symlinks.contains(&norm_cased_executable); + if env.executable.is_none() { + executable_not_in_symlinks = false; + } + + let mut invalid_prefix = env.prefix.clone().unwrap_or_default() != resolved.prefix.clone()?; + if env.prefix.clone().is_none() { + invalid_prefix = false; + } + + let mut invalid_arch = env.arch.clone() != resolved.arch.clone(); + if env.arch.clone().is_none() { + invalid_arch = false; + } + + let invalid_version = are_versions_different( + &resolved.version.clone()?, + &env.version.clone().unwrap_or_default(), + ); + + if invalid_executable + || executable_not_in_symlinks + || invalid_prefix + || invalid_arch + || invalid_version.unwrap_or_default() + { + let event = InaccuratePythonEnvironmentInfo { + category: env.category.clone(), + invalid_executable: Some(invalid_executable), + executable_not_in_symlinks: Some(executable_not_in_symlinks), + invalid_prefix: Some(invalid_prefix), + invalid_version, + invalid_arch: Some(invalid_arch), + }; + warn!( + "Inaccurate Python Environment Info for => \n{}.\nResolved as => \n{}\nIncorrect information => \n{}", + env, resolved, event + ); + // reporter.report_telemetry(TelemetryEvent::InaccuratePythonEnvironmentInfo(event)); + } + Option::Some(()) +} + +fn are_versions_different(actual: &str, expected: &str) -> Option { + let actual = PYTHON_VERSION.captures(actual)?; + let actual = actual.get(1)?.as_str().to_string(); + let expected = PYTHON_VERSION.captures(expected)?; + let expected = expected.get(1)?.as_str().to_string(); + Some(actual != expected) +} diff --git a/crates/pet-windows-registry/src/lib.rs b/crates/pet-windows-registry/src/lib.rs index 527ae198..57909260 100644 --- a/crates/pet-windows-registry/src/lib.rs +++ b/crates/pet-windows-registry/src/lib.rs @@ -4,32 +4,59 @@ #[cfg(windows)] use environments::get_registry_pythons; use pet_conda::{utils::is_conda_env, CondaLocator}; +#[cfg(windows)] +use pet_core::LocatorResult; use pet_core::{python_environment::PythonEnvironment, reporter::Reporter, Locator}; use pet_python_utils::env::PythonEnv; -use std::sync::Arc; +use std::sync::{Arc, RwLock}; mod environments; pub struct WindowsRegistry { #[allow(dead_code)] conda_locator: Arc, + #[allow(dead_code)] + environments: Arc>>>, } impl WindowsRegistry { pub fn from(conda_locator: Arc) -> WindowsRegistry { - WindowsRegistry { conda_locator } + WindowsRegistry { + conda_locator, + environments: Arc::new(RwLock::new(None)), + } + } + #[cfg(windows)] + fn find_with_cache(&self) -> Option { + let mut envs = self.environments.read().unwrap(); + if let Some(environments) = envs.as_ref() { + Some(LocatorResult { + managers: vec![], + environments: environments.clone(), + }) + } else { + drop(envs); + let mut envs = self.environments.write().unwrap(); + let result = get_registry_pythons(&self.conda_locator)?; + envs.replace(result.environments.clone()); + + Some(result) + } } } impl Locator for WindowsRegistry { fn from(&self, env: &PythonEnv) -> Option { + // We need to check this here, as its possible to install + // a Python environment via an Installer that ends up in Windows Registry + // However that environment is a conda environment. if let Some(env_path) = &env.prefix { if is_conda_env(env_path) { return None; } } #[cfg(windows)] - if let Some(result) = get_registry_pythons(&self.conda_locator) { + if let Some(result) = self.find_with_cache() { // Find the same env here for found_env in result.environments { if env.executable.to_str() == env.executable.to_str() { @@ -42,7 +69,11 @@ impl Locator for WindowsRegistry { #[cfg(windows)] fn find(&self, reporter: &dyn Reporter) { - if let Some(result) = get_registry_pythons(&self.conda_locator) { + let mut envs = self.environments.write().unwrap(); + if envs.is_some() { + envs.take(); + } + if let Some(result) = self.find_with_cache() { result .managers .iter() diff --git a/crates/pet-windows-store/src/lib.rs b/crates/pet-windows-store/src/lib.rs index 9226b86b..af63c3ae 100644 --- a/crates/pet-windows-store/src/lib.rs +++ b/crates/pet-windows-store/src/lib.rs @@ -13,6 +13,7 @@ use pet_core::reporter::Reporter; use pet_core::{os_environment::Environment, Locator}; use pet_python_utils::env::PythonEnv; use std::path::Path; +use std::sync::{Arc, RwLock}; pub fn is_windows_app_folder_in_program_files(path: &Path) -> bool { path.to_str().unwrap_or_default().to_string().to_lowercase()[1..] @@ -21,36 +22,69 @@ pub fn is_windows_app_folder_in_program_files(path: &Path) -> bool { pub struct WindowsStore { pub env_vars: EnvVariables, + #[allow(dead_code)] + environments: Arc>>>, } impl WindowsStore { pub fn from(environment: &dyn Environment) -> WindowsStore { WindowsStore { env_vars: EnvVariables::from(environment), + environments: Arc::new(RwLock::new(None)), + } + } + #[cfg(windows)] + fn find_with_cache(&self) -> Option> { + let envs = self + .environments + .read() + .expect("Failed to read environments in windows store"); + if let Some(environments) = envs.as_ref() { + return Some(environments.clone()); + } else { + drop(envs); + let mut envs = self + .environments + .write() + .expect("Failed to read environments in windows store"); + let environments = list_store_pythons(&self.env_vars)?; + envs.replace(environments.clone()); + Some(environments) } } } impl Locator for WindowsStore { - #[allow(unused_variables)] + #[cfg(windows)] fn from(&self, env: &PythonEnv) -> Option { - #[cfg(windows)] - let environments = list_store_pythons(&self.env_vars)?; - #[cfg(windows)] - for found_env in environments { - if let Some(ref python_executable_path) = found_env.executable { - if python_executable_path == &env.executable { - return Some(found_env); + if let Some(environments) = self.find_with_cache() { + for found_env in environments { + if let Some(ref python_executable_path) = found_env.executable { + if python_executable_path == &env.executable { + return Some(found_env); + } } } } None } + #[cfg(unix)] + fn from(&self, _env: &PythonEnv) -> Option { + None + } + #[cfg(windows)] fn find(&self, reporter: &dyn Reporter) { - if let Some(items) = list_store_pythons(&self.env_vars) { - items.iter().for_each(|e| reporter.report_environment(e)) + let mut envs = self.environments.write().unwrap(); + if envs.is_some() { + envs.take(); + } + drop(envs); + if let Some(environments) = self.find_with_cache() { + environments + .iter() + .for_each(|e| reporter.report_environment(e)) } } diff --git a/crates/pet/Cargo.toml b/crates/pet/Cargo.toml index 42a6946c..ca394e00 100644 --- a/crates/pet/Cargo.toml +++ b/crates/pet/Cargo.toml @@ -25,6 +25,7 @@ pet-mac-python-org = { path = "../pet-mac-python-org" } pet-venv = { path = "../pet-venv" } pet-virtualenv = { path = "../pet-virtualenv" } pet-pipenv = { path = "../pet-pipenv" } +pet-telemetry = { path = "../pet-telemetry" } pet-global-virtualenvs = { path = "../pet-global-virtualenvs" } log = "0.4.21" clap = { version = "4.5.4", features = ["derive"] } diff --git a/crates/pet/src/find.rs b/crates/pet/src/find.rs new file mode 100644 index 00000000..e2b8bde4 --- /dev/null +++ b/crates/pet/src/find.rs @@ -0,0 +1,220 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use log::{info, trace, warn}; +use pet_core::os_environment::{Environment, EnvironmentApi}; +use pet_core::reporter::Reporter; +use pet_core::Locator; +use pet_env_var_path::get_search_paths_from_env_variables; +use pet_global_virtualenvs::list_global_virtual_envs_paths; +use pet_python_utils::env::PythonEnv; +use pet_python_utils::executable::{ + find_executable, find_executables, should_search_for_environments_in_path, +}; +use std::fs; +use std::path::PathBuf; +use std::{sync::Arc, thread}; + +use crate::locators::{identify_python_environment_using_locators, Configuration}; + +pub fn find_and_report_envs( + reporter: &dyn Reporter, + configuration: Configuration, + locators: &Arc>>, +) { + info!("Started Refreshing Environments"); + // From settings + let custom_virtual_env_dirs = configuration.custom_virtual_env_dirs.unwrap_or_default(); + let search_paths = configuration.search_paths.unwrap_or_default(); + // 1. Find using known global locators. + thread::scope(|s| { + s.spawn(|| { + thread::scope(|s| { + for locator in locators.iter() { + let locator = locator.clone(); + s.spawn(move || locator.find(reporter)); + } + }); + }); + // Step 2.1: Search in some global locations for virtual envs. + // Step 2.2: And also find in the current PATH variable + s.spawn(|| { + let environment = EnvironmentApi::new(); + let search_paths: Vec = [ + get_search_paths_from_env_variables(&environment), + list_global_virtual_envs_paths( + environment.get_env_var("WORKON_HOME".into()), + environment.get_user_home(), + ), + custom_virtual_env_dirs, + ] + .concat(); + + trace!( + "Searching for environments in global folders: {:?}", + search_paths + ); + + find_python_environments(search_paths, reporter, locators, false) + }); + // Step 3: Find in workspace folders too. + // This can be merged with step 2 as well, as we're only look for environments + // in some folders. + // However we want step 2 to happen faster, as that list of generally much smaller. + // This list of folders generally map to workspace folders + // & users can have a lot of workspace folders and can have a large number fo files/directories + // that could the discovery. + s.spawn(|| { + if search_paths.is_empty() { + return; + } + trace!( + "Searching for environments in custom folders: {:?}", + search_paths + ); + find_python_environments_in_workspace_folders_recursive( + search_paths, + reporter, + locators, + 0, + 1, + ); + }); + }); +} + +fn find_python_environments_in_workspace_folders_recursive( + paths: Vec, + reporter: &dyn Reporter, + locators: &Arc>>, + depth: u32, + max_depth: u32, +) { + thread::scope(|s| { + // Find in cwd + let paths1 = paths.clone(); + s.spawn(|| { + find_python_environments(paths1, reporter, locators, true); + + if depth >= max_depth { + return; + } + + let bin = if cfg!(windows) { "Scripts" } else { "bin" }; + // If the folder has a bin or scripts, then ignore it, its most likely an env. + // I.e. no point looking for python environments in a Python environment. + let paths = paths + .into_iter() + .filter(|p| !p.join(bin).exists()) + .collect::>(); + + for path in paths { + if let Ok(reader) = fs::read_dir(&path) { + let reader = reader + .filter_map(Result::ok) + .filter(|d| d.file_type().is_ok_and(|f| f.is_dir())) + .map(|p| p.path()) + .filter(should_search_for_environments_in_path); + + // Take a batch of 20 items at a time. + let reader = reader.fold(vec![], |f, a| { + let mut f = f; + if f.is_empty() { + f.push(vec![a]); + return f; + } + let last_item = f.last_mut().unwrap(); + if last_item.is_empty() || last_item.len() < 20 { + last_item.push(a); + return f; + } + f.push(vec![a]); + f + }); + + for entry in reader { + find_python_environments_in_workspace_folders_recursive( + entry, + reporter, + locators, + depth + 1, + max_depth, + ); + } + } + } + }); + }); +} + +fn find_python_environments( + paths: Vec, + reporter: &dyn Reporter, + locators: &Arc>>, + is_workspace_folder: bool, +) { + if paths.is_empty() { + return; + } + thread::scope(|s| { + let chunks = if is_workspace_folder { paths.len() } else { 1 }; + for item in paths.chunks(chunks) { + let lst = item.to_vec().clone(); + let locators = locators.clone(); + s.spawn(move || { + find_python_environments_in_paths_with_locators( + lst, + &locators, + reporter, + is_workspace_folder, + ); + }); + } + }); +} + +fn find_python_environments_in_paths_with_locators( + paths: Vec, + locators: &Arc>>, + reporter: &dyn Reporter, + is_workspace_folder: bool, +) { + let executables = if is_workspace_folder { + // If we're in a workspace folder, then we only need to look for bin/python or bin/python.exe + // As workspace folders generally have either virtual env or conda env or the like. + // They will not have environments that will ONLY have a file like `bin/python3`. + // I.e. bin/python will almost always exist. + paths + .iter() + // Paths like /Library/Frameworks/Python.framework/Versions/3.10/bin can end up in the current PATH variable. + // Hence do not just look for files in a bin directory of the path. + .flat_map(|p| find_executable(p)) + .filter_map(Option::Some) + .collect::>() + } else { + paths + .iter() + // Paths like /Library/Frameworks/Python.framework/Versions/3.10/bin can end up in the current PATH variable. + // Hence do not just look for files in a bin directory of the path. + .flat_map(find_executables) + .filter(|p| { + // Exclude python2 on macOS + if std::env::consts::OS == "macos" { + return p.to_str().unwrap_or_default() != "/usr/bin/python2"; + } + true + }) + .collect::>() + }; + + for exe in executables.into_iter() { + let executable = exe.clone(); + let env = PythonEnv::new(exe, None, None); + if let Some(env) = identify_python_environment_using_locators(&env, locators) { + reporter.report_environment(&env); + continue; + } else { + warn!("Unknown Python Env {:?}", executable); + } + } +} diff --git a/crates/pet/src/jsonrpc.rs b/crates/pet/src/jsonrpc.rs index a35d5487..5f55927f 100644 --- a/crates/pet/src/jsonrpc.rs +++ b/crates/pet/src/jsonrpc.rs @@ -1,27 +1,37 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -use log::{error, info}; -use pet::locators::{find_and_report_envs, Configuration}; +use log::{error, trace}; +use pet::resolve::resolve_environment; use pet_conda::Conda; -use pet_core::{os_environment::EnvironmentApi, reporter::Reporter}; +use pet_core::{ + os_environment::EnvironmentApi, python_environment::PythonEnvironment, reporter::Reporter, + Locator, +}; use pet_jsonrpc::{ - send_reply, + send_error, send_reply, server::{start_server, HandlersKeyedByMethodName}, }; -use pet_reporter::jsonrpc; +use pet_reporter::{environment::Environment, jsonrpc}; +use pet_telemetry::report_inaccuracies_identified_after_resolving; use serde::{Deserialize, Serialize}; use serde_json::{self, Value}; use std::{ path::PathBuf, sync::{Arc, RwLock}, - time::SystemTime, + thread, + time::{Duration, SystemTime, SystemTimeError}, +}; + +use crate::{ + find::find_and_report_envs, + locators::{create_locators, Configuration}, }; pub struct Context { reporter: Arc, - conda_locator: Arc, configuration: RwLock, + locators: Arc>>, } pub fn start_jsonrpc_server() { @@ -34,12 +44,13 @@ pub fn start_jsonrpc_server() { let conda_locator = Arc::new(Conda::from(&environment)); let context = Context { reporter: Arc::new(jsonrpc_reporter), - conda_locator, + locators: create_locators(conda_locator.clone()), configuration: RwLock::new(Configuration::default()), }; let mut handlers = HandlersKeyedByMethodName::new(Arc::new(context)); handlers.add_request_handler("refresh", handle_refresh); + handlers.add_request_handler("resolve", handle_resolve); start_server(&handlers) } @@ -49,26 +60,104 @@ pub struct RequestOptions { pub conda_executable: Option, } +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct RefreshResult { + duration: Option, +} + +impl RefreshResult { + pub fn new(duration: Result) -> RefreshResult { + RefreshResult { + duration: duration.ok().map(|d| d.as_millis()), + } + } +} + pub fn handle_refresh(context: Arc, id: u32, params: Value) { - let request_options: RequestOptions = serde_json::from_value(params).unwrap(); - let mut cfg = context.configuration.write().unwrap(); - cfg.search_paths = request_options.search_paths; - cfg.conda_executable = request_options.conda_executable; - drop(cfg); - let config = context.configuration.read().unwrap().clone(); - - info!("Started Refreshing Environments"); - let now = SystemTime::now(); - find_and_report_envs( - context.reporter.as_ref(), - context.conda_locator.clone(), - config, - ); - - if let Ok(duration) = now.elapsed() { - send_reply(id, Some(duration.as_millis())); - } else { - send_reply(id, None::); - error!("Failed to calculate duration"); + match serde_json::from_value::(params.clone()) { + Ok(request_options) => { + let mut cfg = context.configuration.write().unwrap(); + cfg.search_paths = request_options.search_paths; + cfg.conda_executable = request_options.conda_executable; + drop(cfg); + let config = context.configuration.read().unwrap().clone(); + + let now = SystemTime::now(); + find_and_report_envs(context.reporter.as_ref(), config, &context.locators); + send_reply(id, Some(RefreshResult::new(now.elapsed()))); + } + Err(e) => { + send_reply(id, None::); + error!("Failed to parse request options {:?}: {}", params, e); + } + } +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ResolveOptions { + pub executable: PathBuf, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ResolveResult { + environment: Environment, + duration: Option, +} + +impl ResolveResult { + fn new(env: &PythonEnvironment, duration: Result) -> ResolveResult { + ResolveResult { + environment: Environment::from(env), + duration: duration.ok().map(|d| d.as_millis()), + } + } +} + +pub fn handle_resolve(context: Arc, id: u32, params: Value) { + match serde_json::from_value::(params.clone()) { + Ok(request_options) => { + let executable = request_options.executable.clone(); + // Start in a new thread, we can have multiple resolve requests. + thread::spawn(move || { + let now = SystemTime::now(); + trace!("Resolving env {:?}", executable); + if let Some(result) = resolve_environment(&executable, &context.locators) { + if let Some(resolved) = result.resolved { + // Gather telemetry of this resolved env and see what we got wrong. + let _ = report_inaccuracies_identified_after_resolving( + context.reporter.as_ref(), + &result.discovered, + &resolved, + ); + + send_reply(id, Some(ResolveResult::new(&resolved, now.elapsed()))); + } else { + error!( + "Failed to resolve env, returning discovered env {:?}", + executable + ); + send_reply( + id, + Some(ResolveResult::new(&result.discovered, now.elapsed())), + ); + } + } else { + error!("Failed to resolve env {:?}", executable); + send_error( + Some(id), + -4, + format!("Failed to resolve env {:?}", executable), + ); + } + }); + } + Err(e) => { + error!("Failed to parse request {:?}: {}", params, e); + send_error( + Some(id), + -4, + format!("Failed to parse request {:?}: {}", params, e), + ); + } } } diff --git a/crates/pet/src/lib.rs b/crates/pet/src/lib.rs index 6f373f66..9bf7c5c8 100644 --- a/crates/pet/src/lib.rs +++ b/crates/pet/src/lib.rs @@ -1,16 +1,19 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -use locators::Configuration; +use find::find_and_report_envs; +use locators::{create_locators, Configuration}; use pet_conda::Conda; use pet_core::os_environment::EnvironmentApi; use pet_reporter::{self, stdio}; use std::{env, sync::Arc, time::SystemTime}; +pub mod find; pub mod locators; +pub mod resolve; pub fn find_and_report_envs_stdio() { - stdio::initialize_logger(log::LevelFilter::Info); + stdio::initialize_logger(log::LevelFilter::Trace); let now = SystemTime::now(); let reporter = stdio::create_reporter(); @@ -21,7 +24,7 @@ pub fn find_and_report_envs_stdio() { if let Ok(cwd) = env::current_dir() { config.search_paths = Some(vec![cwd]); } - locators::find_and_report_envs(&reporter, conda_locator, config); + find_and_report_envs(&reporter, config, &create_locators(conda_locator)); println!( "Refresh completed in {}ms", now.elapsed().unwrap().as_millis() diff --git a/crates/pet/src/locators.rs b/crates/pet/src/locators.rs index e02cfd16..288ee500 100644 --- a/crates/pet/src/locators.rs +++ b/crates/pet/src/locators.rs @@ -1,472 +1,130 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -use log::{error, info, trace, warn}; +use log::{trace, warn}; use pet_conda::Conda; use pet_core::arch::Architecture; -use pet_core::os_environment::{Environment, EnvironmentApi}; -use pet_core::python_environment::{PythonEnvironmentBuilder, PythonEnvironmentCategory}; -use pet_core::reporter::Reporter; +use pet_core::os_environment::EnvironmentApi; +use pet_core::python_environment::{ + PythonEnvironment, PythonEnvironmentBuilder, PythonEnvironmentCategory, +}; use pet_core::Locator; -use pet_env_var_path::get_search_paths_from_env_variables; -use pet_global_virtualenvs::list_global_virtual_envs_paths; use pet_mac_commandlinetools::MacCmdLineTools; use pet_mac_python_org::MacPythonOrg; use pet_pipenv::PipEnv; use pet_pyenv::PyEnv; use pet_python_utils::env::{PythonEnv, ResolvedPythonEnv}; -use pet_python_utils::executable::{find_executable, find_executables}; -use pet_python_utils::version; use pet_venv::Venv; use pet_virtualenv::VirtualEnv; use pet_virtualenvwrapper::VirtualEnvWrapper; -use std::fs; -use std::ops::Deref; use std::path::PathBuf; -use std::{sync::Arc, thread}; +use std::sync::Arc; #[derive(Debug, Default, Clone)] pub struct Configuration { pub search_paths: Option>, pub conda_executable: Option, + pub custom_virtual_env_dirs: Option>, } -pub fn find_and_report_envs( - reporter: &dyn Reporter, - conda_locator: Arc, - configuration: Configuration, -) { - info!("Started Refreshing Environments"); +pub fn create_locators(conda_locator: Arc) -> Arc>> { + // NOTE: The order of the items matter. + let environment = EnvironmentApi::new(); - let conda_locator1 = conda_locator.clone(); - let conda_locator2 = conda_locator.clone(); - let conda_locator3 = conda_locator.clone(); - let search_paths = configuration.search_paths.unwrap_or_default(); - // 1. Find using known global locators. - thread::scope(|s| { - s.spawn(|| { - find_using_global_finders(conda_locator1, reporter); - }); - // Step 2: Search in some global locations for virtual envs. - s.spawn(|| find_in_global_virtual_env_dirs(reporter)); - // Step 3: Finally find in the current PATH variable - s.spawn(|| { - let environment = EnvironmentApi::new(); - find_python_environments( - conda_locator2, - get_search_paths_from_env_variables(&environment), - reporter, - false, - ) - }); - // Step 4: Find in workspace folders - s.spawn(|| { - if search_paths.is_empty() { - return; - } - trace!( - "Searching for environments in custom folders: {:?}", - search_paths - ); - find_python_environments_in_workspace_folders_recursive( - conda_locator3, - search_paths, - reporter, - 0, - 1, - ); - }); - }); -} + let mut locators: Vec> = vec![]; -#[cfg(windows)] -fn find_using_global_finders(conda_locator: Arc, reporter: &dyn Reporter) { - // Step 1: These environments take precedence over all others. - // As they are very specific and guaranteed to be specific type. - thread::scope(|s| { + // 1. Windows store Python + // 2. Windows registry python + if cfg!(windows) { + #[cfg(windows)] use pet_windows_registry::WindowsRegistry; + #[cfg(windows)] use pet_windows_store::WindowsStore; - // use pet_win - // The order matters, - // Windows store can sometimes get detected via registry locator (but we want to avoid that), - // difficult to repro, but we have see this on Karthiks machine - // Windows registry can contain conda envs (e.g. installing Ananconda will result in registry entries). - // Conda is best done last, as Windows Registry and Pyenv can also contain conda envs, - // Thus lets leave the generic conda locator to last to find all remaining conda envs. - // pyenv can be treated as a virtualenvwrapper environment, hence virtualenvwrapper needs to be detected first - let conda_locator1 = conda_locator.clone(); - let conda_locator2 = conda_locator.clone(); - let conda_locator3 = conda_locator.clone(); - - // 1. windows store - s.spawn(|| { - let environment = EnvironmentApi::new(); - WindowsStore::from(&environment).find(reporter) - }); - // 2. windows registry - s.spawn(|| WindowsRegistry::from(conda_locator1).find(reporter)); - // 3. virtualenvwrapper - s.spawn(|| { - let environment = EnvironmentApi::new(); - VirtualEnvWrapper::from(&environment).find(reporter) - }); - // 4. pyenv - s.spawn(|| { - let environment = EnvironmentApi::new(); - PyEnv::from(&environment, conda_locator2).find(reporter) - }); - // 5. conda - s.spawn(move || conda_locator3.find(reporter)); - }); -} - -#[cfg(unix)] -fn find_using_global_finders(conda_locator: Arc, reporter: &dyn Reporter) { - // Step 1: These environments take precedence over all others. - // As they are very specific and guaranteed to be specific type. - - thread::scope(|s| { - // The order matters, - // pyenv can be treated as a virtualenvwrapper environment, hence virtualenvwrapper needs to be detected first - // Homebrew can happen anytime - // Conda is best done last, as pyenv can also contain conda envs, - // Thus lets leave the generic conda locator to last to find all remaining conda envs. - + #[cfg(windows)] + locators.push(Arc::new(WindowsStore::from(&environment))); + #[cfg(windows)] + locators.push(Arc::new(WindowsRegistry::from(conda_locator.clone()))) + } + // 3. Pyenv Python + locators.push(Arc::new(PyEnv::from(&environment, conda_locator.clone()))); + // 4. Homebrew Python + if cfg!(unix) { + #[cfg(unix)] use pet_homebrew::Homebrew; - - let conda_locator1 = conda_locator.clone(); - let conda_locator2 = conda_locator.clone(); - // 1. virtualenvwrapper - s.spawn(|| { - let environment = EnvironmentApi::new(); - VirtualEnvWrapper::from(&environment).find(reporter) - }); - // 2. pyenv - s.spawn(|| { - let environment = EnvironmentApi::new(); - PyEnv::from(&environment, conda_locator1).find(reporter) - }); - // 3. homebrew - s.spawn(|| { - let environment = EnvironmentApi::new(); - Homebrew::from(&environment).find(reporter) - }); - // 4. conda - s.spawn(move || conda_locator2.find(reporter)); - // 5. Mac Global Python & CommandLineTools Python (xcode) - s.spawn(move || { - if std::env::consts::OS == "macos" { - MacCmdLineTools::new().find(reporter); - MacPythonOrg::new().find(reporter); - } - }); - }); -} - -fn find_in_global_virtual_env_dirs(reporter: &dyn Reporter) { - #[cfg(unix)] - use pet_homebrew::Homebrew; - - let custom_virtual_env_dirs: Vec = vec![]; - - // Step 1: These environments take precedence over all others. - // As they are very specific and guaranteed to be specific type. - - let environment = EnvironmentApi::new(); - let virtualenv_locator = VirtualEnv::new(); - let venv_locator = Venv::new(); - let virtualenvwrapper = VirtualEnvWrapper::from(&environment); - let pipenv_locator = PipEnv::from(&environment); - #[cfg(unix)] - let homebrew_locator = Homebrew::from(&environment); - - let venv_type_locators = vec![ - Box::new(pipenv_locator) as Box, - Box::new(virtualenvwrapper) as Box, - Box::new(venv_locator) as Box, - Box::new(virtualenv_locator) as Box, - ]; - - // Find python envs in custom locations - let envs_from_global_locations: Vec = [ - list_global_virtual_envs_paths( - environment.get_env_var("WORKON_HOME".into()), - environment.get_user_home(), - ), - custom_virtual_env_dirs, - ] - .concat(); - - // Step 2: Search in some global locations for virtual envs. - for env_path in envs_from_global_locations { - if let Some(executable) = find_executable(&env_path) { - let mut env = PythonEnv::new(executable.clone(), Some(env_path.clone()), None); - - // Try to get the version from the env directory - // Never use pyvenv.cfg, as this isn't accurate. - env.version = version::from_header_files(&env_path); - - // 1. First must be homebrew, as it is the most specific and supports symlinks - #[cfg(unix)] - if let Some(env) = homebrew_locator.from(&env) { - reporter.report_environment(&env); - continue; - } - - // 3. Finally Check if these are some kind of virtual env or pipenv. - // Pipeenv before virtualenvwrapper as it is more specific. - // Because pipenv environments are also virtualenvwrapper environments. - // Before venv, as all venvs are also virtualenvwrapper environments. - // Before virtualenv as this is more specific. - // All venvs are also virtualenvs environments. - let mut found = false; - for locator in &venv_type_locators { - if let Some(env) = locator.from(&env) { - reporter.report_environment(&env); - found = true; - break; - } - } - if !found { - // We have no idea what this is. - // We have check all of the resolvers. - error!("Unknown Global Virtual Environment: {:?}", env); - } - } + #[cfg(unix)] + let homebrew_locator = Homebrew::from(&environment); + #[cfg(unix)] + locators.push(Arc::new(homebrew_locator)); } -} - -fn find_python_environments_in_workspace_folders_recursive( - conda_locator: Arc, - paths: Vec, - reporter: &dyn Reporter, - depth: u32, - max_depth: u32, -) { - thread::scope(|s| { - // Find in cwd - let conda_locator1 = conda_locator.clone(); - let paths1 = paths.clone(); - s.spawn(|| { - find_python_environments(conda_locator1, paths1, reporter, true); - - if depth >= max_depth { - return; - } - - let bin = if cfg!(windows) { "Scripts" } else { "bin" }; - // if this is bin or scripts, then we should not go into it. - // This is because the parent of this would have been discovered above. - let paths = paths - .into_iter() - .filter(|p| !p.join(bin).exists()) - .collect::>(); - - for path in paths { - let path = path.clone(); - let conda_locator2 = conda_locator.clone(); - if let Ok(reader) = fs::read_dir(&path) { - let reader = reader - .filter_map(Result::ok) - .map(|p| p.path()) - .filter(|p| p.is_dir()); - - // Take a batch of 20 items at a time. - let reader = reader.fold(vec![], |f, a| { - let mut f = f; - if f.is_empty() { - f.push(vec![a]); - return f; - } - let last_item = f.last_mut().unwrap(); - if last_item.is_empty() || last_item.len() < 20 { - last_item.push(a); - return f; - } - f.push(vec![a]); - f - }); - - for entry in reader { - find_python_environments_in_workspace_folders_recursive( - conda_locator2.clone(), - entry, - reporter, - depth + 1, - max_depth, - ); - } - } - } - }); - }); -} - -fn find_python_environments( - conda_locator: Arc, - paths: Vec, - reporter: &dyn Reporter, - is_workspace_folder: bool, -) { - if paths.is_empty() { - return; + // 5. Conda Python + locators.push(conda_locator); + // 6. Support for Virtual Envs + // The order of these matter. + // Basically PipEnv is a superset of VirtualEnvWrapper, which is a superset of Venv, which is a superset of VirtualEnv. + locators.push(Arc::new(PipEnv::from(&environment))); + locators.push(Arc::new(VirtualEnvWrapper::from(&environment))); + locators.push(Arc::new(Venv::new())); + // VirtualEnv is the most generic, hence should be the last. + locators.push(Arc::new(VirtualEnv::new())); + + // 7. Global Mac Python + // 8. CommandLineTools Python (xcode) + if std::env::consts::OS == "macos" { + locators.push(Arc::new(MacCmdLineTools::new())); + locators.push(Arc::new(MacPythonOrg::new())); } - thread::scope(|s| { - // Step 1: These environments take precedence over all others. - // As they are very specific and guaranteed to be specific type. - - let environment = EnvironmentApi::new(); - let virtualenv_locator = VirtualEnv::new(); - let venv_locator = Venv::new(); - let virtualenvwrapper = VirtualEnvWrapper::from(&environment); - let pipenv_locator = PipEnv::from(&environment); - - let mut all_locators: Vec> = vec![]; - - // First check if this is a known - // 1. Windows store Python - // 2. or Windows registry python - // Note: If we're looking in workspace folders, we should not look in the registry or store. - // As its impossible for windows store or registry exes to be in workspace folders. - if !is_workspace_folder && cfg!(windows) { - #[cfg(windows)] - use pet_windows_registry::WindowsRegistry; - #[cfg(windows)] - use pet_windows_store::WindowsStore; - #[cfg(windows)] - all_locators.push(Arc::new(WindowsStore::from(&environment))); - #[cfg(windows)] - let conda_locator1 = conda_locator.clone(); - #[cfg(windows)] - all_locators.push(Arc::new(WindowsRegistry::from(conda_locator1))) - } - // 3. Check if this is Pyenv Python - let conda_locator1 = conda_locator.clone(); - all_locators.push(Arc::new(PyEnv::from(&environment, conda_locator1))); - // 4. Check if this is Homebrew Python - // Note: If we're looking in workspace folders, we should not look in the registry or store. - // As its impossible for windows store or registry exes to be in workspace folders. - if !is_workspace_folder && cfg!(unix) { - #[cfg(unix)] - use pet_homebrew::Homebrew; - #[cfg(unix)] - let homebrew_locator = Homebrew::from(&environment); - #[cfg(unix)] - all_locators.push(Arc::new(homebrew_locator)); - } - // 5. Check if this is Conda Python - all_locators.push(conda_locator); - // 6. Finally check if this is some kind of a virtual env - all_locators.push(Arc::new(pipenv_locator)); - all_locators.push(Arc::new(virtualenvwrapper)); - all_locators.push(Arc::new(venv_locator)); - all_locators.push(Arc::new(virtualenv_locator)); - // Note: If we're looking in workspace folders, then no point trying to identify a - // Workspace environment as a global one - if !is_workspace_folder && std::env::consts::OS == "macos" { - // 7. Possible this is some global Mac Python environment. - all_locators.push(Arc::new(MacCmdLineTools::new())); - all_locators.push(Arc::new(MacPythonOrg::new())); - } - let all_locators = Arc::new(all_locators); - let chunks = if is_workspace_folder { paths.len() } else { 1 }; - for item in paths.chunks(chunks) { - let lst = item.to_vec().clone(); - let all_locators = all_locators.clone(); - s.spawn(move || { - find_python_environments_in_paths_with_locators( - lst, - all_locators, - reporter, - is_workspace_folder, - ); - }); - } - }); + Arc::new(locators) } -fn find_python_environments_in_paths_with_locators( - paths: Vec, - all_locators: Arc>>, - reporter: &dyn Reporter, - is_workspace_folder: bool, -) { - let executables = if is_workspace_folder { - // If we're in a workspace folder, then we only need to look for python or python.exe - // As this is most likely a virtual env or conda env or the like. - paths - .iter() - // Paths like /Library/Frameworks/Python.framework/Versions/3.10/bin can end up in the current PATH variable. - // Hence do not just look for files in a bin directory of the path. - .flat_map(|p| find_executable(p)) - .filter_map(Option::Some) - .collect::>() - } else { - paths - .iter() - // Paths like /Library/Frameworks/Python.framework/Versions/3.10/bin can end up in the current PATH variable. - // Hence do not just look for files in a bin directory of the path. - .flat_map(find_executables) - .filter(|p| { - // Exclude python2 on macOS - if std::env::consts::OS == "macos" { - return p.to_str().unwrap_or_default() != "/usr/bin/python2"; - } - true - }) - .collect::>() - }; +pub fn identify_python_environment_using_locators( + env: &PythonEnv, + locators: &[Arc], +) -> Option { + let executable = env.executable.clone(); + if let Some(env) = locators + .iter() + .fold(None, |e, loc| if e.is_some() { e } else { loc.from(env) }) + { + return Some(env); + } - for exe in executables.into_iter() { - let executable = exe.clone(); - let env = PythonEnv::new(exe, None, None); - let locators = all_locators.as_ref().deref(); + // Yikes, we have no idea what this is. + // Lets get the actual interpreter info and try to figure this out. + // We try to get the interpreter info, hoping that the real exe returned might be identifiable. + if let Some(resolved_env) = ResolvedPythonEnv::from(&executable) { + let env = resolved_env.to_python_env(); if let Some(env) = locators .iter() .fold(None, |e, loc| if e.is_some() { e } else { loc.from(&env) }) { - reporter.report_environment(&env); - continue; - } - - // Yikes, we have no idea what this is. - // Lets get the actual interpreter info and try to figure this out. - // We try to get the interpreter info, hoping that the real exe returned might be identifiable. - if let Some(resolved_env) = ResolvedPythonEnv::from(&executable) { - let env = resolved_env.to_python_env(); - if let Some(env) = - locators - .iter() - .fold(None, |e, loc| if e.is_some() { e } else { loc.from(&env) }) - { - trace!( - "Unknown Env ({:?}) in Path resolved as {:?}", - executable, - env.category - ); - // TODO: Telemetry point. - // As we had to spawn earlier. - reporter.report_environment(&env); - } else { - // We have no idea what this is. - // We have check all of the resolvers. - // Telemetry point, failed to identify env here. - warn!( - "Unknown Env ({:?}) in Path resolved as {:?} and reported as Unknown", - executable, resolved_env - ); - let env = PythonEnvironmentBuilder::new(PythonEnvironmentCategory::Unknown) - .executable(Some(resolved_env.executable)) - .prefix(Some(resolved_env.prefix)) - .arch(Some(if resolved_env.is64_bit { - Architecture::X64 - } else { - Architecture::X86 - })) - .version(Some(resolved_env.version)) - .build(); - reporter.report_environment(&env); - } + trace!( + "Unknown Env ({:?}) in Path resolved as {:?}", + executable, + env.category + ); + // TODO: Telemetry point. + // As we had to spawn earlier. + return Some(env); + } else { + // We have no idea what this is. + // We have check all of the resolvers. + // Telemetry point, failed to identify env here. + warn!( + "Unknown Env ({:?}) in Path resolved as {:?} and reported as Unknown", + executable, resolved_env + ); + let env = PythonEnvironmentBuilder::new(PythonEnvironmentCategory::Unknown) + .executable(Some(resolved_env.executable)) + .prefix(Some(resolved_env.prefix)) + .arch(Some(if resolved_env.is64_bit { + Architecture::X64 + } else { + Architecture::X86 + })) + .version(Some(resolved_env.version)) + .build(); + return Some(env); } } + None } diff --git a/crates/pet/src/main.rs b/crates/pet/src/main.rs index 448c9b25..fd8b492d 100644 --- a/crates/pet/src/main.rs +++ b/crates/pet/src/main.rs @@ -5,7 +5,9 @@ use clap::{Parser, Subcommand}; use jsonrpc::start_jsonrpc_server; use pet::find_and_report_envs_stdio; +mod find; mod jsonrpc; +mod locators; #[derive(Parser, Debug)] #[command(version, about, long_about = None)] diff --git a/crates/pet/src/resolve.rs b/crates/pet/src/resolve.rs new file mode 100644 index 00000000..f8fc7b8b --- /dev/null +++ b/crates/pet/src/resolve.rs @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use std::{path::PathBuf, sync::Arc}; + +use log::warn; +use pet_core::{arch::Architecture, python_environment::PythonEnvironment, Locator}; +use pet_python_utils::env::{PythonEnv, ResolvedPythonEnv}; + +use crate::locators::identify_python_environment_using_locators; + +pub struct ResolvedEnvironment { + pub discovered: PythonEnvironment, + pub resolved: Option, +} + +pub fn resolve_environment( + executable: &PathBuf, + locators: &Arc>>, +) -> Option { + // First check if this is a known environment + let env = PythonEnv::new(executable.to_owned(), None, None); + if let Some(env) = identify_python_environment_using_locators(&env, locators) { + // Ok we got the environment. + // Now try to resolve this fully, by spawning python. + if let Some(ref executable) = env.executable { + if let Some(info) = ResolvedPythonEnv::from(executable) { + let discovered = env.clone(); + let mut resolved = env.clone(); + let mut symlinks = resolved.symlinks.clone().unwrap_or_default(); + + symlinks.push(info.executable.clone()); + symlinks.append(&mut info.symlink.clone().unwrap_or_default()); + resolved.version = Some(info.version); + resolved.prefix = Some(info.prefix); + resolved.arch = Some(if info.is64_bit { + Architecture::X64 + } else { + Architecture::X86 + }); + + Some(ResolvedEnvironment { + discovered, + resolved: Some(resolved), + }) + } else { + Some(ResolvedEnvironment { + discovered: env, + resolved: None, + }) + } + } else { + warn!("Unknown Python Env {:?} resolved as {:?}", executable, env); + Some(ResolvedEnvironment { + discovered: env, + resolved: None, + }) + } + } else { + warn!("Unknown Python Env {:?}", executable); + None + } +} diff --git a/crates/pet/tests/ci_test.rs b/crates/pet/tests/ci_test.rs index 604e9e75..bbd6d21f 100644 --- a/crates/pet/tests/ci_test.rs +++ b/crates/pet/tests/ci_test.rs @@ -22,18 +22,21 @@ mod common; #[allow(dead_code)] // We should detect the conda install along with the base env fn verify_validity_of_discovered_envs() { - use std::{sync::Arc, thread}; - - use pet::locators; + use pet::{find::find_and_report_envs, locators::create_locators}; use pet_conda::Conda; use pet_core::os_environment::EnvironmentApi; use pet_reporter::test; + use std::{sync::Arc, thread}; let reporter = test::create_reporter(); let environment = EnvironmentApi::new(); let conda_locator = Arc::new(Conda::from(&environment)); - locators::find_and_report_envs(&reporter, conda_locator, Default::default()); + find_and_report_envs( + &reporter, + Default::default(), + &create_locators(conda_locator), + ); let result = reporter.get_result(); let environments = result.environments; @@ -57,7 +60,7 @@ fn verify_validity_of_discovered_envs() { #[allow(dead_code)] // On linux we create a virtualenvwrapper environment named `venv_wrapper_env1` fn check_if_virtualenvwrapper_exists() { - use pet::locators; + use pet::{find::find_and_report_envs, locators::create_locators}; use pet_conda::Conda; use pet_core::os_environment::EnvironmentApi; use pet_reporter::test; @@ -67,7 +70,12 @@ fn check_if_virtualenvwrapper_exists() { let environment = EnvironmentApi::new(); let conda_locator = Arc::new(Conda::from(&environment)); - locators::find_and_report_envs(&reporter, conda_locator, Default::default()); + find_and_report_envs( + &reporter, + Default::default(), + &create_locators(conda_locator), + ); + let result = reporter.get_result(); let environments = result.environments; @@ -95,7 +103,7 @@ fn check_if_virtualenvwrapper_exists() { #[allow(dead_code)] // On linux we create a virtualenvwrapper environment named `venv_wrapper_env1` fn check_if_pyenv_virtualenv_exists() { - use pet::locators; + use pet::{find::find_and_report_envs, locators::create_locators}; use pet_conda::Conda; use pet_core::os_environment::EnvironmentApi; use pet_reporter::test; @@ -105,7 +113,12 @@ fn check_if_pyenv_virtualenv_exists() { let environment = EnvironmentApi::new(); let conda_locator = Arc::new(Conda::from(&environment)); - locators::find_and_report_envs(&reporter, conda_locator, Default::default()); + find_and_report_envs( + &reporter, + Default::default(), + &create_locators(conda_locator), + ); + let result = reporter.get_result(); let environments = result.environments;