From e4438e941a6cc9f1ca97d2c1e2fbb37f8f34b856 Mon Sep 17 00:00:00 2001 From: itowlson Date: Tue, 23 Jul 2024 11:18:26 +1200 Subject: [PATCH 1/2] Validate against target environments during build Signed-off-by: itowlson --- Cargo.lock | 157 ++++++- Cargo.toml | 1 + crates/build/Cargo.toml | 1 + crates/build/src/lib.rs | 105 ++++- crates/build/src/manifest.rs | 125 +++++- crates/compose/src/lib.rs | 111 +++-- crates/environments/Cargo.toml | 42 ++ crates/environments/src/environment.rs | 388 +++++++++++++++++ .../src/environment/definition.rs | 165 +++++++ .../src/environment/env_loader.rs | 336 +++++++++++++++ .../environments/src/environment/lockfile.rs | 210 +++++++++ crates/environments/src/lib.rs | 217 ++++++++++ crates/environments/src/loader.rs | 255 +++++++++++ .../environments/tests/simple-wit/world.wit | 35 ++ crates/loader/src/lib.rs | 2 + crates/loader/src/local.rs | 402 ++++++++++-------- crates/manifest/src/compat.rs | 1 + crates/manifest/src/schema/v2.rs | 30 +- src/commands/build.rs | 26 +- src/commands/registry.rs | 4 +- src/commands/up.rs | 4 +- src/commands/up/app_source.rs | 4 +- 22 files changed, 2384 insertions(+), 237 deletions(-) create mode 100644 crates/environments/Cargo.toml create mode 100644 crates/environments/src/environment.rs create mode 100644 crates/environments/src/environment/definition.rs create mode 100644 crates/environments/src/environment/env_loader.rs create mode 100644 crates/environments/src/environment/lockfile.rs create mode 100644 crates/environments/src/lib.rs create mode 100644 crates/environments/src/loader.rs create mode 100644 crates/environments/tests/simple-wit/world.wit diff --git a/Cargo.lock b/Cargo.lock index 0c461a2521..381664bd5d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5102,6 +5102,7 @@ checksum = "4edc8853320c2a0dab800fbda86253c8938f6ea88510dc92c5f1ed20e794afc1" dependencies = [ "cfg-if", "miette-derive 7.2.0", + "serde", "thiserror 1.0.69", "unicode-width 0.1.14", ] @@ -5590,6 +5591,30 @@ dependencies = [ "unicase", ] +[[package]] +name = "oci-distribution" +version = "0.11.0" +source = "git+https://github.com/fermyon/oci-distribution?rev=7e4ce9be9bcd22e78a28f06204931f10c44402ba#7e4ce9be9bcd22e78a28f06204931f10c44402ba" +dependencies = [ + "bytes", + "chrono", + "futures-util", + "http 1.1.0", + "http-auth", + "jwt 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.5.0", + "olpc-cjson", + "regex", + "reqwest 0.12.9", + "serde", + "serde_json", + "sha2", + "thiserror 1.0.69", + "tokio", + "tracing", + "unicase", +] + [[package]] name = "oci-spec" version = "0.7.1" @@ -7910,6 +7935,7 @@ dependencies = [ "anyhow", "serde", "spin-common", + "spin-environments", "spin-manifest", "subprocess", "terminal", @@ -7961,6 +7987,7 @@ dependencies = [ "spin-build", "spin-common", "spin-doctor", + "spin-environments", "spin-factor-outbound-networking", "spin-http", "spin-loader", @@ -8085,6 +8112,41 @@ dependencies = [ "ui-testing", ] +[[package]] +name = "spin-environments" +version = "3.4.0-pre0" +dependencies = [ + "anyhow", + "async-trait", + "bytes", + "chrono", + "futures", + "futures-util", + "id-arena", + "indexmap 2.7.1", + "oci-distribution 0.11.0 (git+https://github.com/fermyon/oci-distribution?rev=7e4ce9be9bcd22e78a28f06204931f10c44402ba)", + "semver", + "serde", + "serde_json", + "spin-common", + "spin-componentize", + "spin-compose", + "spin-loader", + "spin-manifest", + "spin-serde", + "tokio", + "toml", + "tracing", + "wac-parser", + "wac-resolver", + "wac-types", + "wasm-pkg-client", + "wasmparser 0.230.0", + "wit-component 0.230.0", + "wit-encoder", + "wit-parser 0.230.0", +] + [[package]] name = "spin-expressions" version = "3.4.0-pre0" @@ -8548,7 +8610,7 @@ dependencies = [ "docker_credential", "futures-util", "itertools 0.14.0", - "oci-distribution", + "oci-distribution 0.11.0 (git+https://github.com/fermyon/oci-distribution?rev=7b291a39f74d1a3c9499d934a56cae6580fc8e37)", "reqwest 0.12.9", "serde", "serde_json", @@ -10117,6 +10179,49 @@ dependencies = [ "wasmparser 0.202.0", ] +[[package]] +name = "wac-parser" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "616ec0c4f63641fa095b4a551263fe35a15c72c9680b650b8f08f70db0fdbd19" +dependencies = [ + "anyhow", + "id-arena", + "indexmap 2.7.1", + "log", + "logos", + "miette 7.2.0", + "semver", + "serde", + "thiserror 1.0.69", + "wac-graph", + "wasm-encoder 0.202.0", + "wasm-metadata 0.202.0", + "wasmparser 0.202.0", +] + +[[package]] +name = "wac-resolver" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "688c4c9482d68574aec10b72fa26429d0f91c5065ecbcf38a3146caf9f0bd7db" +dependencies = [ + "anyhow", + "futures", + "indexmap 2.7.1", + "log", + "miette 7.2.0", + "semver", + "thiserror 1.0.69", + "tokio", + "wac-parser", + "wac-types", + "warg-client", + "warg-crypto", + "warg-protocol", + "wit-component 0.202.0", +] + [[package]] name = "wac-types" version = "0.6.1" @@ -11588,6 +11693,25 @@ dependencies = [ "bitflags 2.6.0", ] +[[package]] +name = "wit-component" +version = "0.202.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c836b1fd9932de0431c1758d8be08212071b6bba0151f7bac826dbc4312a2a9" +dependencies = [ + "anyhow", + "bitflags 2.6.0", + "indexmap 2.7.1", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder 0.202.0", + "wasm-metadata 0.202.0", + "wasmparser 0.202.0", + "wit-parser 0.202.0", +] + [[package]] name = "wit-component" version = "0.224.1" @@ -11646,6 +11770,37 @@ dependencies = [ "wit-parser 0.235.0", ] +[[package]] +name = "wit-encoder" +version = "0.229.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52b22120872fbeea51d41381dbd5c91395a9c8ceec77102d49a0bc6b503984ed" +dependencies = [ + "id-arena", + "pretty_assertions", + "semver", + "serde", + "wit-parser 0.229.0", +] + +[[package]] +name = "wit-parser" +version = "0.202.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "744237b488352f4f27bca05a10acb79474415951c450e52ebd0da784c1df2bcc" +dependencies = [ + "anyhow", + "id-arena", + "indexmap 2.7.1", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser 0.202.0", +] + [[package]] name = "wit-parser" version = "0.224.1" diff --git a/Cargo.toml b/Cargo.toml index 007bf72062..717fc55588 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,6 +55,7 @@ spin-app = { path = "crates/app" } spin-build = { path = "crates/build" } spin-common = { path = "crates/common" } spin-doctor = { path = "crates/doctor" } +spin-environments = { path = "crates/environments" } spin-factor-outbound-networking = { path = "crates/factor-outbound-networking" } spin-http = { path = "crates/http" } spin-loader = { path = "crates/loader" } diff --git a/crates/build/Cargo.toml b/crates/build/Cargo.toml index aa698b566c..29eb585ab1 100644 --- a/crates/build/Cargo.toml +++ b/crates/build/Cargo.toml @@ -8,6 +8,7 @@ edition = { workspace = true } anyhow = { workspace = true } serde = { workspace = true } spin-common = { path = "../common" } +spin-environments = { path = "../environments" } spin-manifest = { path = "../manifest" } subprocess = "0.2" terminal = { path = "../terminal" } diff --git a/crates/build/src/lib.rs b/crates/build/src/lib.rs index c122d183e6..2242536666 100644 --- a/crates/build/src/lib.rs +++ b/crates/build/src/lib.rs @@ -16,31 +16,87 @@ use subprocess::{Exec, Redirection}; use crate::manifest::component_build_configs; /// If present, run the build command of each component. -pub async fn build(manifest_file: &Path, component_ids: &[String]) -> Result<()> { - let (components, manifest_err) = - component_build_configs(manifest_file) - .await - .with_context(|| { - format!( - "Cannot read manifest file from {}", - quoted_path(manifest_file) - ) - })?; +pub async fn build( + manifest_file: &Path, + component_ids: &[String], + target_checks: TargetChecking, + cache_root: Option, +) -> Result<()> { + let build_info = component_build_configs(manifest_file) + .await + .with_context(|| { + format!( + "Cannot read manifest file from {}", + quoted_path(manifest_file) + ) + })?; let app_dir = parent_dir(manifest_file)?; - let build_result = build_components(component_ids, components, app_dir); + let build_result = build_components(component_ids, build_info.components(), &app_dir); - if let Some(e) = manifest_err { + // Emit any required warnings now, so that they don't bury any errors. + if let Some(e) = build_info.load_error() { + // The manifest had errors. We managed to attempt a build anyway, but we want to + // let the user know about them. terminal::warn!("The manifest has errors not related to the Wasm component build. Error details:\n{e:#}"); + // Checking deployment targets requires a healthy manifest (because trigger types etc.), + // if any of these were specified, warn they are being skipped. + let should_have_checked_targets = + target_checks.check() && build_info.has_deployment_targets(); + if should_have_checked_targets { + terminal::warn!( + "The manifest error(s) prevented Spin from checking the deployment targets." + ); + } + } + + // If the build failed, exit with an error at this point. + build_result?; + + let Some(manifest) = build_info.manifest() else { + // We can't proceed to checking (because that needs a full healthy manifest), and we've + // already emitted any necessary warning, so quit. + return Ok(()); + }; + + if target_checks.check() { + let application = spin_environments::ApplicationToValidate::new( + manifest.clone(), + manifest_file.parent().unwrap(), + ) + .await + .context("unable to load application for checking against deployment targets")?; + let target_validation = spin_environments::validate_application_against_environment_ids( + &application, + build_info.deployment_targets(), + cache_root.clone(), + &app_dir, + ) + .await + .context("unable to check if the application is compatible with deployment targets")?; + + if !target_validation.is_ok() { + for error in target_validation.errors() { + terminal::error!("{error}"); + } + anyhow::bail!("All components built successfully, but one or more was incompatible with one or more of the deployment targets."); + } } - build_result + Ok(()) +} + +/// Run all component build commands, using the default options (build all +/// components, perform target checking). We run a "default build" in several +/// places and this centralises the logic of what such a "default build" means. +pub async fn build_default(manifest_file: &Path, cache_root: Option) -> Result<()> { + build(manifest_file, &[], TargetChecking::Check, cache_root).await } fn build_components( component_ids: &[String], components: Vec, - app_dir: PathBuf, + app_dir: &Path, ) -> Result<(), anyhow::Error> { let components_to_build = if component_ids.is_empty() { components @@ -70,7 +126,7 @@ fn build_components( components_to_build .into_iter() - .map(|c| build_component(c, &app_dir)) + .map(|c| build_component(c, app_dir)) .collect::, _>>()?; terminal::step!("Finished", "building all Spin components"); @@ -159,6 +215,21 @@ fn construct_workdir(app_dir: &Path, workdir: Option>) -> Resul Ok(cwd) } +/// Specifies target environment checking behaviour +pub enum TargetChecking { + /// The build should check that all components are compatible with all target environments. + Check, + /// The build should not check target environments. + Skip, +} + +impl TargetChecking { + /// Should the build check target environments? + fn check(&self) -> bool { + matches!(self, Self::Check) + } +} + #[cfg(test)] mod tests { use super::*; @@ -171,6 +242,8 @@ mod tests { #[tokio::test] async fn can_load_even_if_trigger_invalid() { let bad_trigger_file = test_data_root().join("bad_trigger.toml"); - build(&bad_trigger_file, &[]).await.unwrap(); + build(&bad_trigger_file, &[], TargetChecking::Skip, None) + .await + .unwrap(); } } diff --git a/crates/build/src/manifest.rs b/crates/build/src/manifest.rs index 2fcd68dadb..db570e4145 100644 --- a/crates/build/src/manifest.rs +++ b/crates/build/src/manifest.rs @@ -4,37 +4,120 @@ use std::{collections::BTreeMap, path::Path}; use spin_manifest::{schema::v2, ManifestVersion}; +#[allow(clippy::large_enum_variant)] // only ever constructed once +pub enum ManifestBuildInfo { + Loadable { + components: Vec, + deployment_targets: Vec, + manifest: spin_manifest::schema::v2::AppManifest, + }, + Unloadable { + components: Vec, + has_deployment_targets: bool, + load_error: spin_manifest::Error, + }, +} + +impl ManifestBuildInfo { + pub fn components(&self) -> Vec { + match self { + Self::Loadable { components, .. } => components.clone(), + Self::Unloadable { components, .. } => components.clone(), + } + } + + pub fn load_error(&self) -> Option<&spin_manifest::Error> { + match self { + Self::Loadable { .. } => None, + Self::Unloadable { load_error, .. } => Some(load_error), + } + } + + pub fn deployment_targets(&self) -> &[spin_manifest::schema::v2::TargetEnvironmentRef] { + match self { + Self::Loadable { + deployment_targets, .. + } => deployment_targets, + Self::Unloadable { .. } => &[], + } + } + + pub fn has_deployment_targets(&self) -> bool { + match self { + Self::Loadable { + deployment_targets, .. + } => !deployment_targets.is_empty(), + Self::Unloadable { + has_deployment_targets, + .. + } => *has_deployment_targets, + } + } + + pub fn manifest(&self) -> Option<&spin_manifest::schema::v2::AppManifest> { + match self { + Self::Loadable { manifest, .. } => Some(manifest), + Self::Unloadable { .. } => None, + } + } +} + /// Returns a map of component IDs to [`v2::ComponentBuildConfig`]s for the /// given (v1 or v2) manifest path. If the manifest cannot be loaded, the /// function attempts fallback: if fallback succeeds, result is Ok but the load error /// is also returned via the second part of the return value tuple. -pub async fn component_build_configs( - manifest_file: impl AsRef, -) -> Result<(Vec, Option)> { +pub async fn component_build_configs(manifest_file: impl AsRef) -> Result { let manifest = spin_manifest::manifest_from_file(&manifest_file); match manifest { - Ok(manifest) => Ok((build_configs_from_manifest(manifest), None)), - Err(e) => fallback_load_build_configs(&manifest_file) - .await - .map(|bc| (bc, Some(e))), + Ok(mut manifest) => { + spin_manifest::normalize::normalize_manifest(&mut manifest); + let components = build_configs_from_manifest(&manifest); + let deployment_targets = deployment_targets_from_manifest(&manifest); + Ok(ManifestBuildInfo::Loadable { + components, + deployment_targets, + manifest, + }) + } + Err(load_error) => { + // The manifest didn't load, but the problem might not be build-affecting. + // Try to fall back by parsing out only the bits we need. And if something + // goes wrong with the fallback, give up and return the original manifest load + // error. + let Ok(components) = fallback_load_build_configs(&manifest_file).await else { + return Err(load_error.into()); + }; + let Ok(has_deployment_targets) = has_deployment_targets(&manifest_file).await else { + return Err(load_error.into()); + }; + Ok(ManifestBuildInfo::Unloadable { + components, + has_deployment_targets, + load_error, + }) + } } } fn build_configs_from_manifest( - mut manifest: spin_manifest::schema::v2::AppManifest, + manifest: &spin_manifest::schema::v2::AppManifest, ) -> Vec { - spin_manifest::normalize::normalize_manifest(&mut manifest); - manifest .components - .into_iter() + .iter() .map(|(id, c)| ComponentBuildInfo { id: id.to_string(), - build: c.build, + build: c.build.clone(), }) .collect() } +fn deployment_targets_from_manifest( + manifest: &spin_manifest::schema::v2::AppManifest, +) -> Vec { + manifest.application.targets.clone() +} + async fn fallback_load_build_configs( manifest_file: impl AsRef, ) -> Result> { @@ -57,7 +140,23 @@ async fn fallback_load_build_configs( }) } -#[derive(Deserialize)] +async fn has_deployment_targets(manifest_file: impl AsRef) -> Result { + let manifest_text = tokio::fs::read_to_string(manifest_file).await?; + Ok(match ManifestVersion::detect(&manifest_text)? { + ManifestVersion::V1 => false, + ManifestVersion::V2 => { + let table: toml::value::Table = toml::from_str(&manifest_text)?; + table + .get("application") + .and_then(|a| a.as_table()) + .and_then(|t| t.get("targets")) + .and_then(|arr| arr.as_array()) + .is_some_and(|arr| !arr.is_empty()) + } + }) +} + +#[derive(Clone, Deserialize)] pub struct ComponentBuildInfo { #[serde(default)] pub id: String, diff --git a/crates/compose/src/lib.rs b/crates/compose/src/lib.rs index 3728ee9346..93a0fa39cc 100644 --- a/crates/compose/src/lib.rs +++ b/crates/compose/src/lib.rs @@ -1,7 +1,7 @@ use anyhow::Context; use indexmap::IndexMap; use semver::Version; -use spin_app::locked::{self, InheritConfiguration, LockedComponent, LockedComponentDependency}; +use spin_app::locked::InheritConfiguration as LockedInheritConfiguration; use spin_common::{ui::quoted_path, url::parse_file_url}; use spin_serde::{DependencyName, KebabId}; use std::collections::BTreeMap; @@ -29,18 +29,72 @@ use wac_graph::{CompositionGraph, NodeId}; /// composition graph into a byte array and return it. pub async fn compose( loader: &L, - component: &LockedComponent, + component: &L::Component, ) -> Result, ComposeError> { Composer::new(loader).compose(component).await } +/// A Spin component dependency. This abstracts over the metadata associated with the +/// dependency. The abstraction allows both manifest and lockfile types to participate in composition. +#[async_trait::async_trait] +pub trait DependencyLike { + fn inherit(&self) -> InheritConfiguration; + fn export(&self) -> &Option; +} + +pub enum InheritConfiguration { + All, + Some(Vec), +} + +/// A Spin component. This abstracts over the list of dependencies for the component. +/// The abstraction allows both manifest and lockfile types to participate in composition. +#[async_trait::async_trait] +pub trait ComponentLike { + type Dependency: DependencyLike; + + fn dependencies( + &self, + ) -> impl std::iter::ExactSizeIterator; + fn id(&self) -> &str; +} + +#[async_trait::async_trait] +impl ComponentLike for spin_app::locked::LockedComponent { + type Dependency = spin_app::locked::LockedComponentDependency; + + fn dependencies( + &self, + ) -> impl std::iter::ExactSizeIterator { + self.dependencies.iter() + } + + fn id(&self) -> &str { + &self.id + } +} + +#[async_trait::async_trait] +impl DependencyLike for spin_app::locked::LockedComponentDependency { + fn inherit(&self) -> InheritConfiguration { + match &self.inherit { + LockedInheritConfiguration::All => InheritConfiguration::All, + LockedInheritConfiguration::Some(cfgs) => InheritConfiguration::Some(cfgs.clone()), + } + } + + fn export(&self) -> &Option { + &self.export + } +} + /// This trait is used to load component source code from a locked component source across various embdeddings. #[async_trait::async_trait] pub trait ComponentSourceLoader { - async fn load_component_source( - &self, - source: &locked::LockedComponentSource, - ) -> anyhow::Result>; + type Component: ComponentLike; + type Dependency: DependencyLike; + async fn load_component_source(&self, source: &Self::Component) -> anyhow::Result>; + async fn load_dependency_source(&self, source: &Self::Dependency) -> anyhow::Result>; } /// A ComponentSourceLoader that loads component sources from the filesystem. @@ -48,9 +102,21 @@ pub struct ComponentSourceLoaderFs; #[async_trait::async_trait] impl ComponentSourceLoader for ComponentSourceLoaderFs { - async fn load_component_source( - &self, - source: &locked::LockedComponentSource, + type Component = spin_app::locked::LockedComponent; + type Dependency = spin_app::locked::LockedComponentDependency; + + async fn load_component_source(&self, source: &Self::Component) -> anyhow::Result> { + Self::load_from_locked_source(&source.source).await + } + + async fn load_dependency_source(&self, source: &Self::Dependency) -> anyhow::Result> { + Self::load_from_locked_source(&source.source).await + } +} + +impl ComponentSourceLoaderFs { + async fn load_from_locked_source( + source: &spin_app::locked::LockedComponentSource, ) -> anyhow::Result> { let source = source .content @@ -129,19 +195,19 @@ struct Composer<'a, L> { } impl<'a, L: ComponentSourceLoader> Composer<'a, L> { - async fn compose(mut self, component: &LockedComponent) -> Result, ComposeError> { + async fn compose(mut self, component: &L::Component) -> Result, ComposeError> { let source = self .loader - .load_component_source(&component.source) + .load_component_source(component) .await .map_err(ComposeError::PrepareError)?; - if component.dependencies.is_empty() { + if component.dependencies().len() == 0 { return Ok(source); } let (world_id, instantiation_id) = self - .register_package(&component.id, None, source) + .register_package(component.id(), None, source) .map_err(ComposeError::PrepareError)?; let prepared = self.prepare_dependencies(world_id, component).await?; @@ -180,7 +246,7 @@ impl<'a, L: ComponentSourceLoader> Composer<'a, L> { async fn prepare_dependencies( &mut self, world_id: WorldId, - component: &LockedComponent, + component: &L::Component, ) -> Result, ComposeError> { let imports = self.graph.types()[world_id].imports.clone(); @@ -188,7 +254,7 @@ impl<'a, L: ComponentSourceLoader> Composer<'a, L> { let mut mappings: BTreeMap> = BTreeMap::new(); - for (dependency_name, dependency) in &component.dependencies { + for (dependency_name, dependency) in component.dependencies() { let mut matched = Vec::new(); for import_name in &import_keys { @@ -201,7 +267,7 @@ impl<'a, L: ComponentSourceLoader> Composer<'a, L> { if matched.is_empty() { return Err(ComposeError::UnmatchedDependencyName { - component_id: component.id.clone(), + component_id: component.id().to_owned(), dependency_name: dependency_name.clone(), }); } @@ -225,7 +291,7 @@ impl<'a, L: ComponentSourceLoader> Composer<'a, L> { if !conflicts.is_empty() { return Err(ComposeError::DependencyConflicts { - component_id: component.id.clone(), + component_id: component.id().to_owned(), conflicts: conflicts .into_iter() .map(|(import_name, infos)| { @@ -330,19 +396,16 @@ impl<'a, L: ComponentSourceLoader> Composer<'a, L> { async fn register_dependency( &mut self, dependency_name: DependencyName, - dependency: &LockedComponentDependency, + dependency: &L::Dependency, ) -> anyhow::Result { - let mut dependency_source = self - .loader - .load_component_source(&dependency.source) - .await?; + let mut dependency_source = self.loader.load_dependency_source(dependency).await?; let package_name = match &dependency_name { DependencyName::Package(name) => name.package.to_string(), DependencyName::Plain(name) => name.to_string(), }; - match &dependency.inherit { + match dependency.inherit() { InheritConfiguration::Some(configurations) => { if configurations.is_empty() { // Configuration inheritance is disabled, apply deny_all adapter @@ -363,7 +426,7 @@ impl<'a, L: ComponentSourceLoader> Composer<'a, L> { manifest_name: dependency_name, instantiation_id, world_id, - export_name: dependency.export.clone(), + export_name: dependency.export().clone(), }) } diff --git a/crates/environments/Cargo.toml b/crates/environments/Cargo.toml new file mode 100644 index 0000000000..189fc7d086 --- /dev/null +++ b/crates/environments/Cargo.toml @@ -0,0 +1,42 @@ +[package] +name = "spin-environments" +version = { workspace = true } +authors = { workspace = true } +edition = { workspace = true } + +[dependencies] +anyhow = { workspace = true } +async-trait = "0.1" +bytes = "1.1" +chrono = { workspace = true } +futures = "0.3" +futures-util = "0.3" +id-arena = "2" +indexmap = "2" +oci-distribution = { git = "https://github.com/fermyon/oci-distribution", rev = "7e4ce9be9bcd22e78a28f06204931f10c44402ba" } +semver = "1" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +spin-common = { path = "../common" } +spin-componentize = { path = "../componentize" } +spin-compose = { path = "../compose" } +spin-loader = { path = "../loader" } +spin-manifest = { path = "../manifest" } +spin-serde = { path = "../serde" } +toml = { workspace = true } +tokio = { version = "1.23", features = ["fs"] } +tracing = { workspace = true } +wac-parser = "0.6.0" +wac-resolver = "0.6.0" +wac-types = "0.6.0" +wasm-pkg-client = { workspace = true } +wasmparser = { workspace = true } +wit-component = { workspace = true } +wit-parser = { workspace = true } + +[dev-dependencies] +wit-component = { workspace = true, features = ["dummy-module"] } +wit-encoder = "0.229" + +[lints] +workspace = true diff --git a/crates/environments/src/environment.rs b/crates/environments/src/environment.rs new file mode 100644 index 0000000000..2ca7e1e4ad --- /dev/null +++ b/crates/environments/src/environment.rs @@ -0,0 +1,388 @@ +use std::{collections::HashMap, path::Path}; + +use anyhow::Context; +use spin_common::ui::quoted_path; +use spin_manifest::schema::v2::TargetEnvironmentRef; + +mod definition; +mod env_loader; +mod lockfile; + +use definition::WorldName; + +/// A fully realised deployment environment, e.g. Spin 2.7, +/// SpinKube 3.1, Fermyon Cloud. The `TargetEnvironment` provides a mapping +/// from the Spin trigger types supported in the environment to the Component Model worlds +/// supported by that trigger type. (A trigger type may support more than one world, +/// for example when it supports multiple versions of the Spin or WASI interfaces.) +pub struct TargetEnvironment { + name: String, + trigger_worlds: HashMap, + unknown_trigger: UnknownTrigger, +} + +impl TargetEnvironment { + /// Loads the specified list of environments. This fetches all required + /// environment definitions from their references, and then chases packages + /// references until the entire target environment is fully loaded. + /// The function also caches registry references in the application directory, + /// to avoid loading from the network when the app is validated again. + pub async fn load_all( + env_ids: &[TargetEnvironmentRef], + cache_root: Option, + app_dir: &std::path::Path, + ) -> anyhow::Result> { + env_loader::load_environments(env_ids, cache_root, app_dir).await + } + + /// The environment name for UI purposes + pub fn name(&self) -> &str { + &self.name + } + + /// Returns true if the given trigger type can run in this environment. + pub fn supports_trigger_type(&self, trigger_type: &TriggerType) -> bool { + self.unknown_trigger.allows(trigger_type) || self.trigger_worlds.contains_key(trigger_type) + } + + /// Lists all worlds supported for the given trigger type in this environment. + pub fn worlds(&self, trigger_type: &TriggerType) -> &CandidateWorlds { + self.trigger_worlds + .get(trigger_type) + .or_else(|| self.unknown_trigger.worlds()) + .unwrap_or(NO_WORLDS) + } +} + +/// How a `TargetEnvironment` should validate components associated with trigger types +/// not listed in the/ environment definition. This is used for best-effort validation in +/// extensible environments. +/// +/// For example, a "forgiving" definition of Spin CLI environment would +/// validate that components associated with `cron` or `sqs` triggers adhere +/// to the platform world, even though it cannot validate that the exports are correct +/// or that the plugins are installed or up to date. This can result in failure at +/// runtime, but that may be better than refusing to let cron jobs run! +/// +/// On the other hand, the SpinKube environment rejects unknown triggers +/// because SpinKube does not allow arbitrary triggers to be linked at +/// runtime: the set of triggers is static for a given version. +enum UnknownTrigger { + /// Components for unknown trigger types fail validation. + Deny, + /// Components for unknown trigger types pass validation if they + /// conform to (at least) one of the listed worlds. + Allow(CandidateWorlds), +} + +impl UnknownTrigger { + fn allows(&self, _trigger_type: &TriggerType) -> bool { + matches!(self, Self::Allow(_)) + } + + fn worlds(&self) -> Option<&CandidateWorlds> { + match self { + Self::Deny => None, + Self::Allow(cw) => Some(cw), + } + } +} + +/// The set of worlds that a particular trigger type (in a given environment) +/// can accept. For example, the Spin 3.2 CLI `http` trigger accepts various +/// versions of the `spin:up/http-trigger` world. +/// +/// A component will pass target validation if it conforms to +/// at least one of these worlds. +#[derive(Default)] +pub struct CandidateWorlds { + worlds: Vec, +} + +impl<'a> IntoIterator for &'a CandidateWorlds { + type Item = &'a CandidateWorld; + + type IntoIter = std::slice::Iter<'a, CandidateWorld>; + + fn into_iter(self) -> Self::IntoIter { + self.worlds.iter() + } +} + +const NO_WORLDS: &CandidateWorlds = &CandidateWorlds { worlds: vec![] }; + +/// A WIT world; specifically, a WIT world provided by a Spin host, against which +/// a component can be validated. +pub struct CandidateWorld { + world: WorldName, + package: wit_parser::Package, + package_bytes: Vec, +} + +impl std::fmt::Display for CandidateWorld { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.world.fmt(f) + } +} + +impl CandidateWorld { + /// Namespaced but unversioned package name (e.g. spin:up) + pub fn package_namespaced_name(&self) -> String { + format!("{}:{}", self.package.name.namespace, self.package.name.name) + } + + /// The package version for the environment package. + pub fn package_version(&self) -> Option<&semver::Version> { + self.package.name.version.as_ref() + } + + /// The Wasm-encoded bytes of the environment package. + pub fn package_bytes(&self) -> &[u8] { + &self.package_bytes + } + + fn from_package_bytes(world: &WorldName, bytes: Vec) -> anyhow::Result { + let decoded = wit_component::decode(&bytes) + .with_context(|| format!("Failed to decode package for environment {world}"))?; + let package_id = decoded.package(); + let package = decoded + .resolve() + .packages + .get(package_id) + .with_context(|| { + format!("The {world} package is invalid (no package for decoded package ID)") + })? + .clone(); + + Ok(Self { + world: world.to_owned(), + package, + package_bytes: bytes, + }) + } + + fn from_decoded_wasm( + world: &WorldName, + source: &Path, + decoded: wit_parser::decoding::DecodedWasm, + ) -> anyhow::Result { + let package_id = decoded.package(); + let package = decoded + .resolve() + .packages + .get(package_id) + .with_context(|| { + format!( + "The {} environment is invalid (no package for decoded package ID)", + quoted_path(source) + ) + })? + .clone(); + + let bytes = wit_component::encode(decoded.resolve(), package_id)?; + + Ok(Self { + world: world.to_owned(), + package, + package_bytes: bytes, + }) + } +} + +pub(super) fn is_versioned(env_id: &str) -> bool { + env_id.contains(':') +} + +pub type TriggerType = String; + +#[cfg(test)] +mod test { + use super::*; + + use std::path::PathBuf; + + const SIMPLE_WIT_DIR: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/simple-wit"); + + /// Construct a CandidateWorlds that matches only the named" world. + fn load_simple_world(wit_path: &Path, world: &str) -> CandidateWorlds { + let mut resolve = wit_parser::Resolve::default(); + let (id, _) = resolve + .push_dir(wit_path) + .expect("should have pushed WIT dir"); + let package_bytes = + wit_component::encode(&resolve, id).expect("should have encoded world package"); + + let world_name = WorldName::try_from(world.to_owned()).unwrap(); + let simple_world = CandidateWorld::from_package_bytes(&world_name, package_bytes) + .expect("should have loaded world package"); + + CandidateWorlds { + worlds: vec![simple_world], + } + } + + /// Build an environment using the given WIT that maps the "s" trigger + /// to the "spin:test/simple@1.0.0" world (and denies all other triggers). + fn target_simple_world(wit_path: &Path) -> TargetEnvironment { + let candidate_worlds = load_simple_world(wit_path, "spin:test/simple@1.0.0"); + + TargetEnvironment { + name: "test".to_owned(), + trigger_worlds: [("s".to_owned(), candidate_worlds)].into_iter().collect(), + unknown_trigger: UnknownTrigger::Deny, + } + } + + /// Build an environment using the given WIT that maps all triggers to + /// the "spin:test/simple-import-only@1.0.0" world. (This isn't a very realistic example + /// because a fallback world would usually be imports-only.) + fn target_import_only_forgiving(wit_path: &Path) -> TargetEnvironment { + let candidate_worlds = load_simple_world(wit_path, "spin:test/simple-import-only@1.0.0"); + + TargetEnvironment { + name: "test".to_owned(), + trigger_worlds: [].into_iter().collect(), + unknown_trigger: UnknownTrigger::Allow(candidate_worlds), + } + } + + #[tokio::test] + async fn can_validate_component() { + let wit_path = PathBuf::from(SIMPLE_WIT_DIR); + + let wit_text = tokio::fs::read_to_string(wit_path.join("world.wit")) + .await + .unwrap(); + let wasm = generate_dummy_component(&wit_text, "spin:test/simple@1.0.0"); + + let env = target_simple_world(&wit_path); + + assert!(env.supports_trigger_type(&"s".to_owned())); + assert!(!env.supports_trigger_type(&"t".to_owned())); + + let component = crate::ComponentToValidate::new("scomp", "scomp.wasm", wasm); + let errs = + crate::validate_component_against_environments(&[env], &"s".to_owned(), &component) + .await; + assert!( + errs.is_empty(), + "{}", + errs.iter() + .map(|e| e.to_string()) + .collect::>() + .join("\n") + ); + } + + #[tokio::test] + async fn can_validate_component_for_unknown_trigger() { + let wit_path = PathBuf::from(SIMPLE_WIT_DIR); + + let wit_text = tokio::fs::read_to_string(wit_path.join("world.wit")) + .await + .unwrap(); + // The actual component has an export, although the target world can't check that + let wasm = generate_dummy_component(&wit_text, "spin:test/simple@1.0.0"); + + let env = target_import_only_forgiving(&wit_path); + + // E.g. a plugin trigger that isn't part of the Spin CLI + let non_existent_trigger = "farmer-buckleys-trousers-explode".to_owned(); + + assert!(env.supports_trigger_type(&non_existent_trigger)); + + let component = crate::ComponentToValidate::new("comp", "comp.wasm", wasm); + let errs = crate::validate_component_against_environments( + &[env], + &non_existent_trigger, + &component, + ) + .await; + assert!( + errs.is_empty(), + "{}", + errs.iter() + .map(|e| e.to_string()) + .collect::>() + .join("\n") + ); + } + + #[tokio::test] + async fn unavailable_import_invalidates_component() { + let wit_path = PathBuf::from(SIMPLE_WIT_DIR); + + let wit_text = tokio::fs::read_to_string(wit_path.join("world.wit")) + .await + .unwrap(); + let wasm = generate_dummy_component(&wit_text, "spin:test/not-so-simple@1.0.0"); + + let env = target_simple_world(&wit_path); + + let component = crate::ComponentToValidate::new("nscomp", "nscomp.wasm", wasm); + let errs = + crate::validate_component_against_environments(&[env], &"s".to_owned(), &component) + .await; + assert!(!errs.is_empty()); + + let err = errs[0].to_string(); + assert!( + err.contains("Component nscomp (nscomp.wasm) can't run in environment test"), + "unexpected error {err}" + ); + assert!(err.contains( + "world spin:test/simple@1.0.0 does not provide an import named spin:test/evil@1.0.0" + ), "unexpected error {err}"); + } + + #[tokio::test] + async fn unprovided_export_invalidates_component() { + let wit_path = PathBuf::from(SIMPLE_WIT_DIR); + + let wit_text = tokio::fs::read_to_string(wit_path.join("world.wit")) + .await + .unwrap(); + let wasm = generate_dummy_component(&wit_text, "spin:test/too-darn-simple@1.0.0"); + + let env = target_simple_world(&wit_path); + + let component = crate::ComponentToValidate::new("tdscomp", "tdscomp.wasm", wasm); + let errs = + crate::validate_component_against_environments(&[env], &"s".to_owned(), &component) + .await; + assert!(!errs.is_empty()); + + let err = errs[0].to_string(); + assert!( + err.contains("Component tdscomp (tdscomp.wasm) can't run in environment test"), + "unexpected error {err}" + ); + } + + fn generate_dummy_component(wit: &str, world: &str) -> Vec { + let mut resolve = wit_parser::Resolve::default(); + let package_id = resolve.push_str("test", wit).expect("should parse WIT"); + let world_id = resolve + .select_world(package_id, Some(world)) + .expect("should select world"); + + let mut wasm = wit_component::dummy_module( + &resolve, + world_id, + wit_parser::ManglingAndAbi::Legacy(wit_parser::LiftLowerAbi::Sync), + ); + wit_component::embed_component_metadata( + &mut wasm, + &resolve, + world_id, + wit_component::StringEncoding::UTF8, + ) + .expect("should embed component metadata"); + + let mut encoder = wit_component::ComponentEncoder::default() + .validate(true) + .module(&wasm) + .expect("should set module"); + encoder.encode().expect("should encode component") + } +} diff --git a/crates/environments/src/environment/definition.rs b/crates/environments/src/environment/definition.rs new file mode 100644 index 0000000000..1e439b14cb --- /dev/null +++ b/crates/environments/src/environment/definition.rs @@ -0,0 +1,165 @@ +//! Environment definition types and serialisation (TOML) formats +//! +//! This module does *not* cover loading those definitions from remote +//! sources, or materialising WIT packages from files or registry references - +//! only the types. + +use std::collections::HashMap; + +use anyhow::Context; + +/// An environment definition, usually deserialised from a TOML document. +/// Example: +/// +/// ```ignore +/// # spin-up.3.2.toml +/// [triggers] +/// http = ["spin:up/http-trigger@3.2.0", "spin:up/http-trigger-rc20231018@3.2.0"] +/// redis = ["spin:up/redis-trigger@3.2.0"] +/// ``` +#[derive(Debug, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub struct EnvironmentDefinition { + triggers: HashMap>, + default: Option>, +} + +impl EnvironmentDefinition { + pub fn triggers(&self) -> &HashMap> { + &self.triggers + } + + pub fn default(&self) -> Option<&Vec> { + self.default.as_ref() + } +} + +/// A reference to a world in an [EnvironmentDefinition]. This is formed +/// of a fully qualified (ns:pkg/id) world name, optionally with +/// a location from which to get the package (a registry or WIT directory). +#[derive(Clone, Debug, serde::Deserialize)] +#[serde(untagged, deny_unknown_fields)] +pub enum WorldRef { + DefaultRegistry(WorldName), + Registry { + registry: String, + world: WorldName, + }, + WitDirectory { + path: std::path::PathBuf, + world: WorldName, + }, +} + +/// The qualified name of a world, e.g. spin:up/http-trigger@3.2.0. +/// +/// (Internally it is represented as a PackageName plus unqualified +/// world name, but it stringises to the standard WIT qualified name.) +#[derive(Clone, Debug, serde::Deserialize)] +#[serde(try_from = "String")] +pub struct WorldName { + package: wit_parser::PackageName, + world: String, +} + +impl WorldName { + pub fn package(&self) -> &wit_parser::PackageName { + &self.package + } + + pub fn package_namespaced_name(&self) -> String { + format!("{}:{}", self.package.namespace, self.package.name) + } + + pub fn package_ref(&self) -> anyhow::Result { + let pkg_name = self.package_namespaced_name(); + pkg_name + .parse() + .with_context(|| format!("Environment {pkg_name} is not a valid package name")) + } + + pub fn package_version(&self) -> Option<&semver::Version> { + self.package.version.as_ref() + } +} + +impl TryFrom for WorldName { + type Error = anyhow::Error; + + fn try_from(value: String) -> Result { + use wasmparser::names::{ComponentName, ComponentNameKind}; + + // World qnames have the same syntactic form as interface qnames + let parsed = ComponentName::new(&value, 0)?; + let ComponentNameKind::Interface(itf) = parsed.kind() else { + anyhow::bail!("{value} is not a well-formed world name"); + }; + + let package = wit_parser::PackageName { + namespace: itf.namespace().to_string(), + name: itf.package().to_string(), + version: itf.version(), + }; + + let world = itf.interface().to_string(); + + Ok(Self { package, world }) + } +} + +impl std::fmt::Display for WorldName { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.package.namespace)?; + f.write_str(":")?; + f.write_str(&self.package.name)?; + f.write_str("/")?; + f.write_str(&self.world)?; + + if let Some(v) = self.package.version.as_ref() { + f.write_str("@")?; + f.write_str(&v.to_string())?; + } + + Ok(()) + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn can_parse_versioned_world_name() { + let text = "ns:name/world@1.0.0"; + let w = WorldName::try_from(text.to_owned()).unwrap(); + + assert_eq!("ns", w.package().namespace); + assert_eq!("name", w.package().name); + assert_eq!("ns:name", w.package_namespaced_name()); + assert_eq!("ns", w.package_ref().unwrap().namespace().to_string()); + assert_eq!("name", w.package_ref().unwrap().name().to_string()); + assert_eq!("world", w.world); + assert_eq!( + &semver::Version::new(1, 0, 0), + w.package().version.as_ref().unwrap() + ); + + assert_eq!(text, w.to_string()); + } + + #[test] + fn can_parse_unversioned_world_name() { + let text = "ns:name/world"; + let w = WorldName::try_from("ns:name/world".to_owned()).unwrap(); + + assert_eq!("ns", w.package().namespace); + assert_eq!("name", w.package().name); + assert_eq!("ns:name", w.package_namespaced_name()); + assert_eq!("ns", w.package_ref().unwrap().namespace().to_string()); + assert_eq!("name", w.package_ref().unwrap().name().to_string()); + assert_eq!("world", w.world); + assert!(w.package().version.is_none()); + + assert_eq!(text, w.to_string()); + } +} diff --git a/crates/environments/src/environment/env_loader.rs b/crates/environments/src/environment/env_loader.rs new file mode 100644 index 0000000000..217d063521 --- /dev/null +++ b/crates/environments/src/environment/env_loader.rs @@ -0,0 +1,336 @@ +//! Loading target environments, from a list of references through to +//! a fully realised collection of WIT packages with their worlds and +//! mappings. + +use std::{collections::HashMap, path::Path}; + +use anyhow::{anyhow, Context}; +use futures::future::try_join_all; +use spin_common::ui::quoted_path; +use spin_manifest::schema::v2::TargetEnvironmentRef; + +use super::definition::{EnvironmentDefinition, WorldName, WorldRef}; +use super::lockfile::TargetEnvironmentLockfile; +use super::{is_versioned, CandidateWorld, CandidateWorlds, TargetEnvironment, UnknownTrigger}; + +const DEFAULT_ENV_DEF_REGISTRY_PREFIX: &str = "ghcr.io/spinframework/environments"; +const DEFAULT_PACKAGE_REGISTRY: &str = "spinframework.dev"; + +/// Load all the listed environments from their registries or paths. +/// Registry data will be cached, with a lockfile under `.spin` mapping +/// environment IDs to digests (to allow cache lookup without needing +/// to fetch the digest from the registry). +pub async fn load_environments( + env_ids: &[TargetEnvironmentRef], + cache_root: Option, + app_dir: &std::path::Path, +) -> anyhow::Result> { + if env_ids.is_empty() { + return Ok(Default::default()); + } + + let cache = spin_loader::cache::Cache::new(cache_root) + .await + .context("Unable to create cache")?; + let lockfile_dir = app_dir.join(".spin"); + let lockfile_path = lockfile_dir.join("target-environments.lock"); + + let orig_lockfile: TargetEnvironmentLockfile = tokio::fs::read_to_string(&lockfile_path) + .await + .ok() + .and_then(|s| serde_json::from_str(&s).ok()) + .unwrap_or_default(); + let lockfile = std::sync::Arc::new(tokio::sync::RwLock::new(orig_lockfile.clone())); + + let envs = try_join_all( + env_ids + .iter() + .map(|e| load_environment(e, &cache, &lockfile)), + ) + .await?; + + let final_lockfile = &*lockfile.read().await; + if *final_lockfile != orig_lockfile { + if let Ok(lockfile_json) = serde_json::to_string_pretty(&final_lockfile) { + _ = tokio::fs::create_dir_all(lockfile_dir).await; + _ = tokio::fs::write(&lockfile_path, lockfile_json).await; // failure to update lockfile is not an error + } + } + + Ok(envs) +} + +/// Loads the given `TargetEnvironment` from a registry or directory. +async fn load_environment( + env_id: &TargetEnvironmentRef, + cache: &spin_loader::cache::Cache, + lockfile: &std::sync::Arc>, +) -> anyhow::Result { + match env_id { + TargetEnvironmentRef::DefaultRegistry(id) => { + load_environment_from_registry(DEFAULT_ENV_DEF_REGISTRY_PREFIX, id, cache, lockfile) + .await + } + TargetEnvironmentRef::Registry { registry, id } => { + load_environment_from_registry(registry, id, cache, lockfile).await + } + TargetEnvironmentRef::File { path } => { + load_environment_from_file(path, cache, lockfile).await + } + } +} + +/// Loads a `TargetEnvironment` from the environment definition at the given +/// registry location. The environment and any remote packages it references will be used +/// from cache if available; otherwise, they will be saved to the cache, and the +/// in-memory lockfile object updated. +async fn load_environment_from_registry( + registry: &str, + env_id: &str, + cache: &spin_loader::cache::Cache, + lockfile: &std::sync::Arc>, +) -> anyhow::Result { + let env_def_toml = load_env_def_toml_from_registry(registry, env_id, cache, lockfile).await?; + load_environment_from_toml(env_id, &env_def_toml, cache, lockfile).await +} + +/// Loads a `TargetEnvironment` from the given TOML file. Any remote packages +/// it references will be used from cache if available; otherwise, they will be saved +/// to the cache, and the in-memory lockfile object updated. +async fn load_environment_from_file( + path: &Path, + cache: &spin_loader::cache::Cache, + lockfile: &std::sync::Arc>, +) -> anyhow::Result { + let name = path + .file_stem() + .and_then(|s| s.to_str()) + .map(|s| s.to_owned()) + .unwrap(); + let toml_text = tokio::fs::read_to_string(path).await.with_context(|| { + format!( + "unable to read target environment from {}", + quoted_path(path) + ) + })?; + load_environment_from_toml(&name, &toml_text, cache, lockfile).await +} + +/// Loads a `TargetEnvironment` from the given TOML text. Any remote packages +/// it references will be used from cache if available; otherwise, they will be saved +/// to the cache, and the in-memory lockfile object updated. +async fn load_environment_from_toml( + name: &str, + toml_text: &str, + cache: &spin_loader::cache::Cache, + lockfile: &std::sync::Arc>, +) -> anyhow::Result { + let env: EnvironmentDefinition = toml::from_str(toml_text)?; + + let mut trigger_worlds = HashMap::new(); + + // TODO: parallel all the things + // TODO: this loads _all_ triggers not just the ones we need + for (trigger_type, world_refs) in env.triggers() { + trigger_worlds.insert( + trigger_type.to_owned(), + load_worlds(world_refs, cache, lockfile).await?, + ); + } + + let unknown_trigger = match env.default() { + None => UnknownTrigger::Deny, + Some(world_refs) => UnknownTrigger::Allow(load_worlds(world_refs, cache, lockfile).await?), + }; + + Ok(TargetEnvironment { + name: name.to_owned(), + trigger_worlds, + unknown_trigger, + }) +} + +/// Loads the text (assumed to be TOML) from the environment definition at the given +/// registry location. The environment will be used from cache if available; otherwise, +/// it be saved to the cache, and the in-memory lockfile object updated. +async fn load_env_def_toml_from_registry( + registry: &str, + env_id: &str, + cache: &spin_loader::cache::Cache, + lockfile: &std::sync::Arc>, +) -> anyhow::Result { + if let Some(digest) = lockfile.read().await.env_digest(registry, env_id) { + if let Ok(cache_file) = cache.data_file(digest) { + if let Ok(bytes) = tokio::fs::read(&cache_file).await { + return Ok(String::from_utf8_lossy(&bytes).to_string()); + } + } + } + + let (bytes, digest) = download_env_def_file(registry, env_id) + .await + .with_context(|| format!("downloading target environment {env_id} from {registry}"))?; + + let toml_text = String::from_utf8_lossy(&bytes).to_string(); + + _ = cache.write_data(bytes, &digest).await; + lockfile + .write() + .await + .set_env_digest(registry, env_id, &digest); + + Ok(toml_text) +} + +/// Downloads a single-layer document from the given registry. +/// (You can create a suitable document with e.g. `oras push ghcr.io/my/envs/sample:1.0 sample.toml`.) +/// The image must be publicly accessible (which is *NOT* the default with GHCR). +/// +/// The return value is a tuple of (content, digest). +async fn download_env_def_file(registry: &str, env_id: &str) -> anyhow::Result<(Vec, String)> { + // This implies env_id is in the format spin-up:3.2 + let registry_id = if is_versioned(env_id) { + env_id.to_string() + } else { + // Testing versionless tags with GHCR it didn't work + // TODO: is this expected or am I being a dolt + // TODO: is this a suitable workaround + format!("{env_id}:latest") + }; + + let reference = format!("{registry}/{registry_id}"); + let reference = oci_distribution::Reference::try_from(reference)?; + + let config = oci_distribution::client::ClientConfig::default(); + let client = oci_distribution::client::Client::new(config); + let auth = oci_distribution::secrets::RegistryAuth::Anonymous; + + let (manifest, digest) = client.pull_manifest(&reference, &auth).await?; + + let im = match manifest { + oci_distribution::manifest::OciManifest::Image(im) => im, + oci_distribution::manifest::OciManifest::ImageIndex(_) => { + anyhow::bail!("unexpected registry format for {reference}") + } + }; + + let count = im.layers.len(); + + if count != 1 { + anyhow::bail!("artifact {reference} should have had exactly one layer"); + } + + let the_layer = &im.layers[0]; + let mut out = Vec::with_capacity(the_layer.size.try_into().unwrap_or_default()); + client.pull_blob(&reference, the_layer, &mut out).await?; + + Ok((out, digest)) +} + +async fn load_worlds( + world_refs: &[WorldRef], + cache: &spin_loader::cache::Cache, + lockfile: &std::sync::Arc>, +) -> anyhow::Result { + let mut worlds = vec![]; + + for world_ref in world_refs { + worlds.push(load_world(world_ref, cache, lockfile).await?); + } + + Ok(CandidateWorlds { worlds }) +} + +async fn load_world( + world_ref: &WorldRef, + cache: &spin_loader::cache::Cache, + lockfile: &std::sync::Arc>, +) -> anyhow::Result { + match world_ref { + WorldRef::DefaultRegistry(world) => { + load_world_from_registry(DEFAULT_PACKAGE_REGISTRY, world, cache, lockfile).await + } + WorldRef::Registry { registry, world } => { + load_world_from_registry(registry, world, cache, lockfile).await + } + WorldRef::WitDirectory { path, world } => load_world_from_dir(path, world), + } +} + +fn load_world_from_dir(path: &Path, world: &WorldName) -> anyhow::Result { + let mut resolve = wit_parser::Resolve::default(); + let (pkg_id, _) = resolve.push_dir(path)?; + let decoded = wit_parser::decoding::DecodedWasm::WitPackage(resolve, pkg_id); + CandidateWorld::from_decoded_wasm(world, path, decoded) +} + +/// Loads the given `TargetEnvironment` from the given registry, or +/// from cache if available. If the environment is not in cache, the +/// encoded WIT will be cached, and the in-memory lockfile object +/// updated. +async fn load_world_from_registry( + registry: &str, + world_name: &WorldName, + cache: &spin_loader::cache::Cache, + lockfile: &std::sync::Arc>, +) -> anyhow::Result { + use futures_util::TryStreamExt; + + if let Some(digest) = lockfile + .read() + .await + .package_digest(registry, world_name.package()) + { + if let Ok(cache_file) = cache.wasm_file(digest) { + if let Ok(bytes) = tokio::fs::read(&cache_file).await { + return CandidateWorld::from_package_bytes(world_name, bytes); + } + } + } + + let pkg_name = world_name.package_namespaced_name(); + let pkg_ref = world_name.package_ref()?; + + let wkg_registry: wasm_pkg_client::Registry = registry + .parse() + .with_context(|| format!("Registry {registry} is not a valid registry name"))?; + + let mut wkg_config = wasm_pkg_client::Config::global_defaults().await?; + wkg_config.set_package_registry_override( + pkg_ref, + wasm_pkg_client::RegistryMapping::Registry(wkg_registry), + ); + + let client = wasm_pkg_client::Client::new(wkg_config); + + let package = pkg_name + .to_owned() + .try_into() + .with_context(|| format!("Failed to parse environment name {pkg_name} as package name"))?; + let version = world_name + .package_version() // TODO: surely we can cope with worlds from unversioned packages? surely? + .ok_or_else(|| anyhow!("{world_name} is unversioned: this is not currently supported"))?; + + let release = client + .get_release(&package, version) + .await + .with_context(|| format!("Failed to get {} from registry", world_name.package()))?; + let stm = client + .stream_content(&package, &release) + .await + .with_context(|| format!("Failed to get {} from registry", world_name.package()))?; + let bytes = stm + .try_collect::() + .await + .with_context(|| format!("Failed to get {} from registry", world_name.package()))? + .to_vec(); + + let digest = release.content_digest.to_string(); + _ = cache.write_wasm(&bytes, &digest).await; // Failure to cache is not fatal + lockfile + .write() + .await + .set_package_digest(registry, world_name.package(), &digest); + + CandidateWorld::from_package_bytes(world_name, bytes) +} diff --git a/crates/environments/src/environment/lockfile.rs b/crates/environments/src/environment/lockfile.rs new file mode 100644 index 0000000000..e9019b23a8 --- /dev/null +++ b/crates/environments/src/environment/lockfile.rs @@ -0,0 +1,210 @@ +use std::collections::HashMap; + +use super::is_versioned; + +const DIGEST_TTL_HOURS: i64 = 24; + +/// Serialisation format for the lockfile: registry -> env|pkg -> { name -> digest } +#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct TargetEnvironmentLockfile(HashMap); + +#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +struct Digests { + env: HashMap, + package: HashMap, +} + +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(untagged)] +enum ExpirableDigest { + Forever(String), + Expiring { + digest: String, + correct_at: chrono::DateTime, + }, +} + +impl TargetEnvironmentLockfile { + pub fn env_digest(&self, registry: &str, env_id: &str) -> Option<&str> { + self.0 + .get(registry) + .and_then(|ds| ds.env.get(env_id)) + .and_then(|s| s.current()) + } + + pub fn set_env_digest(&mut self, registry: &str, env_id: &str, digest: &str) { + // If the environment is versioned, we assume it will not change (that is, any changes will + // be reflected as a new version). If the environment is *not* versioned, it represents + // a hosted service which may change over time: allow the cached definition to expire every day or + // so that we do not use a definition that is out of sync with the actual service. + let expirable_digest = if is_versioned(env_id) { + ExpirableDigest::forever(digest) + } else { + ExpirableDigest::expiring(digest) + }; + + match self.0.get_mut(registry) { + Some(ds) => { + ds.env.insert(env_id.to_string(), expirable_digest); + } + None => { + let map = vec![(env_id.to_string(), expirable_digest)] + .into_iter() + .collect(); + let ds = Digests { + env: map, + package: Default::default(), + }; + self.0.insert(registry.to_string(), ds); + } + } + } + + pub fn package_digest( + &self, + registry: &str, + package: &wit_parser::PackageName, + ) -> Option<&str> { + self.0 + .get(registry) + .and_then(|ds| ds.package.get(&package.to_string())) + .map(|s| s.as_str()) + } + + pub fn set_package_digest( + &mut self, + registry: &str, + package: &wit_parser::PackageName, + digest: &str, + ) { + match self.0.get_mut(registry) { + Some(ds) => { + ds.package.insert(package.to_string(), digest.to_string()); + } + None => { + let map = vec![(package.to_string(), digest.to_string())] + .into_iter() + .collect(); + let ds = Digests { + env: Default::default(), + package: map, + }; + self.0.insert(registry.to_string(), ds); + } + } + } +} + +impl ExpirableDigest { + fn current(&self) -> Option<&str> { + match self { + Self::Forever(digest) => Some(digest), + Self::Expiring { digest, correct_at } => { + let now = chrono::Utc::now(); + let time_since = now - correct_at; + if time_since.abs().num_hours() > DIGEST_TTL_HOURS { + None + } else { + Some(digest) + } + } + } + } + + fn forever(digest: &str) -> Self { + Self::Forever(digest.to_string()) + } + + fn expiring(digest: &str) -> Self { + Self::Expiring { + digest: digest.to_string(), + correct_at: chrono::Utc::now(), + } + } +} + +#[cfg(test)] +mod test { + use super::*; + + const DUMMY_REG: &str = "reggy-mc-regface"; + + #[test] + fn versioned_envs_have_no_expiry() { + const TEST_ENV: &str = "my-env:1.0"; + const TEST_DIGEST: &str = "12345"; + + let mut lockfile = TargetEnvironmentLockfile::default(); + lockfile.set_env_digest(DUMMY_REG, TEST_ENV, TEST_DIGEST); + + let json = serde_json::to_value(&lockfile).unwrap(); + + let saved_digest = json + .get(DUMMY_REG) + .and_then(|j| j.get("env")) + .and_then(|j| j.get(TEST_ENV)) + .expect("should have had recorded a digest"); + let saved_digest = saved_digest + .as_str() + .expect("saved digest should have been a string"); + assert_eq!(TEST_DIGEST, saved_digest); + } + + #[test] + fn unversioned_envs_expire() { + const TEST_ENV: &str = "my-env"; + const TEST_DIGEST: &str = "12345"; + + let mut lockfile = TargetEnvironmentLockfile::default(); + lockfile.set_env_digest(DUMMY_REG, TEST_ENV, TEST_DIGEST); + + let json = serde_json::to_value(&lockfile).unwrap(); + + let saved_digest = json + .get(DUMMY_REG) + .and_then(|j| j.get("env")) + .and_then(|j| j.get(TEST_ENV)) + .expect("should have recorded a digest"); + let saved_digest = saved_digest + .as_object() + .expect("saved digest should have been an object"); + assert_eq!(TEST_DIGEST, saved_digest.get("digest").unwrap()); + assert!(saved_digest + .get("correct_at") + .is_some_and(|v| v.is_string())); + } + + #[test] + fn expired_env_digests_are_not_returned() { + const TEST_ENV: &str = "my-env"; + const TEST_DIGEST: &str = "12345"; + + let mut lockfile = TargetEnvironmentLockfile::default(); + lockfile.set_env_digest(DUMMY_REG, TEST_ENV, TEST_DIGEST); + assert_eq!( + TEST_DIGEST, + lockfile + .env_digest(DUMMY_REG, TEST_ENV) + .expect("should have returned env digest") + ); + + // Pass this legit lockfile through JSON and massage the digest date to be old. NEARLY AS OLD AS ME + let mut json = serde_json::to_value(&lockfile).unwrap(); + let digest = json + .get_mut(DUMMY_REG) + .and_then(|j| j.get_mut("env")) + .and_then(|j| j.get_mut(TEST_ENV)) + .expect("should have recorded a digest"); + let digest = digest + .as_object_mut() + .expect("saved digest should have been an object"); + digest.insert( + "correct_at".to_string(), + serde_json::to_value("1969-12-31T01:01:01.001001001Z").unwrap(), + ); + let stale_lockfile: TargetEnvironmentLockfile = serde_json::from_value(json).unwrap(); + + // It should not give us the potentially stale digest + assert!(stale_lockfile.env_digest(DUMMY_REG, TEST_ENV).is_none()); + } +} diff --git a/crates/environments/src/lib.rs b/crates/environments/src/lib.rs new file mode 100644 index 0000000000..5e8371b206 --- /dev/null +++ b/crates/environments/src/lib.rs @@ -0,0 +1,217 @@ +use anyhow::{anyhow, Context}; + +mod environment; +mod loader; + +use environment::{CandidateWorld, CandidateWorlds, TargetEnvironment, TriggerType}; +pub use loader::ApplicationToValidate; +use loader::ComponentToValidate; +use spin_manifest::schema::v2::TargetEnvironmentRef; + +/// The result of validating an application against a list of target environments. +/// If `is_ok` returns true (or equivalently if the `errors` collection is empty), +/// the application passed validation, and can run in all the environments against +/// which it was checked. Otherwise, at least one component cannot run in at least +/// one target environment, and the `errors` collection contains the details. +#[derive(Default)] +pub struct TargetEnvironmentValidation(Vec); + +impl TargetEnvironmentValidation { + pub fn is_ok(&self) -> bool { + self.0.is_empty() + } + + pub fn errors(&self) -> &[anyhow::Error] { + &self.0 + } +} + +/// Validates *all* application components against the list of referenced target enviroments. Each component must conform +/// to *all* environments to pass. +/// +/// If the return value is `Ok(...)`, this means only that we were able to perform the validation. +/// The caller **MUST** still check the returned [TargetEnvironmentValidation] to determine the +/// outcome of validation. +/// +/// If the return value is `Err(...)`, then we weren't able even to attempt validation. +pub async fn validate_application_against_environment_ids( + application: &ApplicationToValidate, + env_ids: &[TargetEnvironmentRef], + cache_root: Option, + app_dir: &std::path::Path, +) -> anyhow::Result { + if env_ids.is_empty() { + return Ok(Default::default()); + } + + let envs = TargetEnvironment::load_all(env_ids, cache_root, app_dir).await?; + validate_application_against_environments(application, &envs).await +} + +/// Validates *all* application components against the list of (realised) target enviroments. Each component must conform +/// to *all* environments to pass. +/// +/// For the slightly funky return type, see [validate_application_against_environment_ids]. +async fn validate_application_against_environments( + application: &ApplicationToValidate, + envs: &[TargetEnvironment], +) -> anyhow::Result { + for trigger_type in application.trigger_types() { + if let Some(env) = envs.iter().find(|e| !e.supports_trigger_type(trigger_type)) { + anyhow::bail!( + "Environment {} does not support trigger type {trigger_type}", + env.name() + ); + } + } + + let components_by_trigger_type = application.components_by_trigger_type().await?; + + let mut errs = vec![]; + + for (trigger_type, component) in components_by_trigger_type { + for component in &component { + errs.extend( + validate_component_against_environments(envs, &trigger_type, component).await, + ); + } + } + + Ok(TargetEnvironmentValidation(errs)) +} + +/// Validates the component against the list of target enviroments. The component must conform +/// to *all* environments to pass. +/// +/// The return value contains the list of validation errors. There may be up to one error per +/// target environment, explaining why the component cannot run in that environment. +/// An empty list means the component has passed validation and is compatible with +/// all target environments. +async fn validate_component_against_environments( + envs: &[TargetEnvironment], + trigger_type: &TriggerType, + component: &ComponentToValidate<'_>, +) -> Vec { + let mut errs = vec![]; + + for env in envs { + let worlds = env.worlds(trigger_type); + if let Some(e) = validate_wasm_against_any_world(env, worlds, component) + .await + .err() + { + errs.push(e); + } + } + + if errs.is_empty() { + tracing::info!( + "Validated component {} {} against all target worlds", + component.id(), + component.source_description() + ); + } + + errs +} + +/// Validates the component against the list of candidate worlds. The component must conform +/// to *at least one* candidate world to pass (since if it can run in one world provided by +/// the target environment, it can run in the target environment). +async fn validate_wasm_against_any_world( + env: &TargetEnvironment, + worlds: &CandidateWorlds, + component: &ComponentToValidate<'_>, +) -> anyhow::Result<()> { + let mut result = Ok(()); + for target_world in worlds { + tracing::debug!( + "Trying component {} {} against target world {target_world}", + component.id(), + component.source_description(), + ); + match validate_wasm_against_world(env, target_world, component).await { + Ok(()) => { + tracing::info!( + "Validated component {} {} against target world {target_world}", + component.id(), + component.source_description(), + ); + return Ok(()); + } + Err(e) => { + // Record the error, but continue in case a different world succeeds + tracing::info!( + "Rejecting component {} {} for target world {target_world} because {e:?}", + component.id(), + component.source_description(), + ); + result = Err(e); + } + } + } + result +} + +async fn validate_wasm_against_world( + env: &TargetEnvironment, + target_world: &CandidateWorld, + component: &ComponentToValidate<'_>, +) -> anyhow::Result<()> { + // Because we are abusing a composition tool to do validation, we have to + // provide a name by which to refer to the component in the dummy composition. + let component_name = "root:component"; + let component_key = wac_types::BorrowedPackageKey::from_name_and_version(component_name, None); + + // wac is going to get the world from the environment package bytes. + // This constructs a key for that mapping. + let env_pkg_name = target_world.package_namespaced_name(); + let env_pkg_key = wac_types::BorrowedPackageKey::from_name_and_version( + &env_pkg_name, + target_world.package_version(), + ); + + let env_name = env.name(); + + let wac_text = format!( + r#" + package validate:component@1.0.0 targets {target_world}; + let c = new {component_name} {{ ... }}; + export c...; + "# + ); + + let doc = wac_parser::Document::parse(&wac_text) + .context("Internal error constructing WAC document for target checking")?; + + let mut packages: indexmap::IndexMap> = + Default::default(); + + packages.insert(env_pkg_key, target_world.package_bytes().to_vec()); + packages.insert(component_key, component.wasm_bytes().to_vec()); + + match doc.resolve(packages) { + Ok(_) => Ok(()), + Err(wac_parser::resolution::Error::TargetMismatch { kind, name, world, .. }) => { + // This one doesn't seem to get hit at the moment - we get MissingTargetExport or ImportNotInTarget instead + Err(anyhow!("Component {} ({}) can't run in environment {env_name} because world {world} expects an {} named {name}", component.id(), component.source_description(), kind.to_string().to_lowercase())) + } + Err(wac_parser::resolution::Error::MissingTargetExport { name, world, .. }) => { + Err(anyhow!("Component {} ({}) can't run in environment {env_name} because world {world} requires an export named {name}, which the component does not provide", component.id(), component.source_description())) + } + Err(wac_parser::resolution::Error::PackageMissingExport { export, .. }) => { + // TODO: The export here seems wrong - it seems to contain the world name rather than the interface name + Err(anyhow!("Component {} ({}) can't run in environment {env_name} because world {target_world} requires an export named {export}, which the component does not provide", component.id(), component.source_description())) + } + Err(wac_parser::resolution::Error::ImportNotInTarget { name, world, .. }) => { + Err(anyhow!("Component {} ({}) can't run in environment {env_name} because world {world} does not provide an import named {name}, which the component requires", component.id(), component.source_description())) + } + Err(wac_parser::resolution::Error::SpreadExportNoEffect { .. }) => { + // We don't have any name info in this case, but it *may* indicate that the component doesn't provide any export at all + Err(anyhow!("Component {} ({}) can't run in environment {env_name} because it requires an export which the component does not provide", component.id(), component.source_description())) + } + Err(e) => { + Err(anyhow!(e)) + }, + } +} diff --git a/crates/environments/src/loader.rs b/crates/environments/src/loader.rs new file mode 100644 index 0000000000..dc31f600a0 --- /dev/null +++ b/crates/environments/src/loader.rs @@ -0,0 +1,255 @@ +//! Loading an application for validation. + +use std::path::Path; + +use anyhow::{anyhow, Context}; +use futures::future::try_join_all; +use spin_common::ui::quoted_path; + +pub(crate) struct ComponentToValidate<'a> { + id: &'a str, + source_description: String, + wasm: Vec, +} + +impl ComponentToValidate<'_> { + pub fn id(&self) -> &str { + self.id + } + + pub fn source_description(&self) -> &str { + &self.source_description + } + + pub fn wasm_bytes(&self) -> &[u8] { + &self.wasm + } + + #[cfg(test)] + pub(crate) fn new(id: &'static str, description: &str, wasm: Vec) -> Self { + Self { + id, + source_description: description.to_owned(), + wasm, + } + } +} + +pub struct ApplicationToValidate { + manifest: spin_manifest::schema::v2::AppManifest, + wasm_loader: spin_loader::WasmLoader, +} + +impl ApplicationToValidate { + pub async fn new( + manifest: spin_manifest::schema::v2::AppManifest, + base_dir: impl AsRef, + ) -> anyhow::Result { + let wasm_loader = + spin_loader::WasmLoader::new(base_dir.as_ref().to_owned(), None, None).await?; + Ok(Self { + manifest, + wasm_loader, + }) + } + + fn component_source<'a>( + &'a self, + trigger: &'a spin_manifest::schema::v2::Trigger, + ) -> anyhow::Result> { + let component_spec = trigger + .component + .as_ref() + .ok_or_else(|| anyhow!("No component specified for trigger {}", trigger.id))?; + let (id, source, dependencies) = match component_spec { + spin_manifest::schema::v2::ComponentSpec::Inline(c) => { + (trigger.id.as_str(), &c.source, &c.dependencies) + } + spin_manifest::schema::v2::ComponentSpec::Reference(r) => { + let id = r.as_ref(); + let Some(component) = self.manifest.components.get(r) else { + anyhow::bail!( + "Component {id} specified for trigger {} does not exist", + trigger.id + ); + }; + (id, &component.source, &component.dependencies) + } + }; + + Ok(ComponentSource { + id, + source, + dependencies: WrappedComponentDependencies::new(dependencies), + }) + } + + pub fn trigger_types(&self) -> impl Iterator { + self.manifest.triggers.keys() + } + + pub fn triggers( + &self, + ) -> impl Iterator)> { + self.manifest.triggers.iter() + } + + pub(crate) async fn components_by_trigger_type( + &self, + ) -> anyhow::Result>)>> { + use futures::FutureExt; + + let components_by_trigger_type_futs = self.triggers().map(|(ty, ts)| { + self.components_for_trigger(ts) + .map(|css| css.map(|css| (ty.to_owned(), css))) + }); + let components_by_trigger_type = try_join_all(components_by_trigger_type_futs) + .await + .context("Failed to prepare components for target environment checking")?; + Ok(components_by_trigger_type) + } + + async fn components_for_trigger<'a>( + &'a self, + triggers: &'a [spin_manifest::schema::v2::Trigger], + ) -> anyhow::Result>> { + let component_futures = triggers.iter().map(|t| self.load_and_resolve_trigger(t)); + try_join_all(component_futures).await + } + + async fn load_and_resolve_trigger<'a>( + &'a self, + trigger: &'a spin_manifest::schema::v2::Trigger, + ) -> anyhow::Result> { + let component = self.component_source(trigger)?; + + let loader = ComponentSourceLoader::new(&self.wasm_loader); + + let wasm = spin_compose::compose(&loader, &component).await.with_context(|| format!("Spin needed to compose dependencies for {} as part of target checking, but composition failed", component.id))?; + + Ok(ComponentToValidate { + id: component.id, + source_description: source_description(component.source), + wasm, + }) + } +} + +struct ComponentSource<'a> { + id: &'a str, + source: &'a spin_manifest::schema::v2::ComponentSource, + dependencies: WrappedComponentDependencies, +} + +struct ComponentSourceLoader<'a> { + wasm_loader: &'a spin_loader::WasmLoader, +} + +impl<'a> ComponentSourceLoader<'a> { + pub fn new(wasm_loader: &'a spin_loader::WasmLoader) -> Self { + Self { wasm_loader } + } +} + +#[async_trait::async_trait] +impl<'a> spin_compose::ComponentSourceLoader for ComponentSourceLoader<'a> { + type Component = ComponentSource<'a>; + type Dependency = WrappedComponentDependency; + async fn load_component_source(&self, source: &Self::Component) -> anyhow::Result> { + let path = self + .wasm_loader + .load_component_source(source.id, source.source) + .await?; + let bytes = tokio::fs::read(&path).await?; + let component = spin_componentize::componentize_if_necessary(&bytes)?; + Ok(component.into()) + } + + async fn load_dependency_source(&self, source: &Self::Dependency) -> anyhow::Result> { + let (path, _) = self + .wasm_loader + .load_component_dependency(&source.name, &source.dependency) + .await?; + let bytes = tokio::fs::read(&path).await?; + let component = spin_componentize::componentize_if_necessary(&bytes)?; + Ok(component.into()) + } +} + +// This exists only to thwart the orphan rule +struct WrappedComponentDependency { + name: spin_serde::DependencyName, + dependency: spin_manifest::schema::v2::ComponentDependency, +} + +// To manage lifetimes around the thwarting of the orphan rule +struct WrappedComponentDependencies { + dependencies: indexmap::IndexMap, +} + +impl WrappedComponentDependencies { + fn new(deps: &spin_manifest::schema::v2::ComponentDependencies) -> Self { + let dependencies = deps + .inner + .clone() + .into_iter() + .map(|(k, v)| { + ( + k.clone(), + WrappedComponentDependency { + name: k, + dependency: v, + }, + ) + }) + .collect(); + Self { dependencies } + } +} + +#[async_trait::async_trait] +impl spin_compose::ComponentLike for ComponentSource<'_> { + type Dependency = WrappedComponentDependency; + + fn dependencies( + &self, + ) -> impl std::iter::ExactSizeIterator + { + self.dependencies.dependencies.iter() + } + + fn id(&self) -> &str { + self.id + } +} + +#[async_trait::async_trait] +impl spin_compose::DependencyLike for WrappedComponentDependency { + fn inherit(&self) -> spin_compose::InheritConfiguration { + // We don't care because this never runs - it is only used to + // verify import satisfaction. Choosing All avoids the compose + // algorithm meddling with it using the deny adapter. + spin_compose::InheritConfiguration::All + } + + fn export(&self) -> &Option { + match &self.dependency { + spin_manifest::schema::v2::ComponentDependency::Version(_) => &None, + spin_manifest::schema::v2::ComponentDependency::Package { export, .. } => export, + spin_manifest::schema::v2::ComponentDependency::Local { export, .. } => export, + spin_manifest::schema::v2::ComponentDependency::HTTP { export, .. } => export, + } + } +} + +fn source_description(source: &spin_manifest::schema::v2::ComponentSource) -> String { + match source { + spin_manifest::schema::v2::ComponentSource::Local(path) => { + format!("file {}", quoted_path(path)) + } + spin_manifest::schema::v2::ComponentSource::Remote { url, .. } => format!("URL {url}"), + spin_manifest::schema::v2::ComponentSource::Registry { package, .. } => { + format!("package {package}") + } + } +} diff --git a/crates/environments/tests/simple-wit/world.wit b/crates/environments/tests/simple-wit/world.wit new file mode 100644 index 0000000000..a726dce9cb --- /dev/null +++ b/crates/environments/tests/simple-wit/world.wit @@ -0,0 +1,35 @@ +package spin:test@1.0.0; + +interface getter { + get: func() -> u32; +} + +interface trigger { + run: func(); +} + +world simple { + import getter; + export trigger; +} + +world simple-import-only { + import getter; +} + +// These worlds and interface are used for constructing components that +// *don't* comply with the 'simple' world. + +interface evil { + cackle: func(); +} + +world not-so-simple { + import getter; + import evil; + export trigger; +} + +world too-darn-simple { + import getter; +} diff --git a/crates/loader/src/lib.rs b/crates/loader/src/lib.rs index ea64bac40d..0142425bfd 100644 --- a/crates/loader/src/lib.rs +++ b/crates/loader/src/lib.rs @@ -23,6 +23,8 @@ mod fs; mod http; mod local; +pub use local::WasmLoader; + /// Maximum number of files to copy (or download) concurrently pub(crate) const MAX_FILE_LOADING_CONCURRENCY: usize = 16; diff --git a/crates/loader/src/local.rs b/crates/loader/src/local.rs index 511202fb76..afe796d64b 100644 --- a/crates/loader/src/local.rs +++ b/crates/loader/src/local.rs @@ -25,8 +25,8 @@ use crate::{cache::Cache, FilesMountStrategy}; pub struct LocalLoader { app_root: PathBuf, files_mount_strategy: FilesMountStrategy, - cache: Cache, - file_loading_permits: Semaphore, + file_loading_permits: std::sync::Arc, + wasm_loader: WasmLoader, } impl LocalLoader { @@ -37,12 +37,14 @@ impl LocalLoader { ) -> Result { let app_root = safe_canonicalize(app_root) .with_context(|| format!("Invalid manifest dir `{}`", app_root.display()))?; + let file_loading_permits = + std::sync::Arc::new(Semaphore::new(crate::MAX_FILE_LOADING_CONCURRENCY)); Ok(Self { - app_root, + app_root: app_root.clone(), files_mount_strategy, - cache: Cache::new(cache_root).await?, // Limit concurrency to avoid hitting system resource limits - file_loading_permits: Semaphore::new(crate::MAX_FILE_LOADING_CONCURRENCY), + file_loading_permits: file_loading_permits.clone(), + wasm_loader: WasmLoader::new(app_root, cache_root, Some(file_loading_permits)).await?, }) } @@ -268,74 +270,15 @@ impl LocalLoader { dependency_name: DependencyName, dependency: v2::ComponentDependency, ) -> Result { - let (content, export) = match dependency { - v2::ComponentDependency::Version(version) => { - let version = semver::VersionReq::parse(&version).with_context(|| format!("Component dependency {dependency_name:?} specifies an invalid semantic version requirement ({version:?}) for its package version"))?; - - // This `unwrap()` should be OK because we've already validated - // this form of dependency requires a package name, i.e. the - // dependency name is not a kebab id. - let package = dependency_name.package().unwrap(); - - let content = self.load_registry_source(None, package, &version).await?; - (content, None) - } - v2::ComponentDependency::Package { - version, - registry, - package, - export, - } => { - let version = semver::VersionReq::parse(&version).with_context(|| format!("Component dependency {dependency_name:?} specifies an invalid semantic version requirement ({version:?}) for its package version"))?; - - let package = match package { - Some(package) => { - package.parse().with_context(|| format!("Component dependency {dependency_name:?} specifies an invalid package name ({package:?})"))? - } - None => { - // This `unwrap()` should be OK because we've already validated - // this form of dependency requires a package name, i.e. the - // dependency name is not a kebab id. - dependency_name - .package() - .cloned() - .unwrap() - } - }; - - let registry = match registry { - Some(registry) => { - registry - .parse() - .map(Some) - .with_context(|| format!("Component dependency {dependency_name:?} specifies an invalid registry name ({registry:?})"))? - } - None => None, - }; - - let content = self - .load_registry_source(registry.as_ref(), &package, &version) - .await?; - (content, export) - } - v2::ComponentDependency::Local { path, export } => { - let content = file_content_ref(self.app_root.join(path))?; - (content, export) - } - v2::ComponentDependency::HTTP { - url, - digest, - export, - } => { - let content = self.load_http_source(&url, &digest).await?; - (content, export) - } - }; + let (content, export) = self + .wasm_loader + .load_component_dependency(&dependency_name, &dependency) + .await?; Ok(LockedComponentDependency { source: LockedComponentSource { content_type: "application/wasm".into(), - content, + content: file_content_ref(content)?, }, export, inherit: if inherit_configuration { @@ -353,116 +296,16 @@ impl LocalLoader { component_id: &KebabId, source: v2::ComponentSource, ) -> Result { - let content = match source { - v2::ComponentSource::Local(path) => file_content_ref(self.app_root.join(path))?, - v2::ComponentSource::Remote { url, digest } => { - self.load_http_source(&url, &digest).await? - } - v2::ComponentSource::Registry { - registry, - package, - version, - } => { - let version = semver::Version::parse(&version).with_context(|| format!("Component {component_id} specifies an invalid semantic version ({version:?}) for its package version"))?; - let version_req = format!("={version}").parse().expect("version"); - - self.load_registry_source(registry.as_ref(), &package, &version_req) - .await? - } - }; + let path = self + .wasm_loader + .load_component_source(component_id.as_ref(), &source) + .await?; Ok(LockedComponentSource { content_type: "application/wasm".into(), - content, + content: file_content_ref(path)?, }) } - // Load a Wasm source from the given HTTP ContentRef source URL and - // return a ContentRef an absolute path to the local copy. - async fn load_http_source(&self, url: &str, digest: &str) -> Result { - ensure!( - digest.starts_with("sha256:"), - "invalid `digest` {digest:?}; must start with 'sha256:'" - ); - let path = if let Ok(cached_path) = self.cache.wasm_file(digest) { - cached_path - } else { - let _loading_permit = self.file_loading_permits.acquire().await?; - - self.cache.ensure_dirs().await?; - let dest = self.cache.wasm_path(digest); - verified_download( - url, - digest, - &dest, - crate::http::DestinationConvention::ContentIndexed, - ) - .await - .with_context(|| format!("Error fetching source URL {url:?}"))?; - dest - }; - file_content_ref(path) - } - - async fn load_registry_source( - &self, - registry: Option<&wasm_pkg_client::Registry>, - package: &wasm_pkg_client::PackageRef, - version: &semver::VersionReq, - ) -> Result { - let mut client_config = wasm_pkg_client::Config::global_defaults().await?; - - if let Some(registry) = registry.cloned() { - let mapping = wasm_pkg_client::RegistryMapping::Registry(registry); - client_config.set_package_registry_override(package.clone(), mapping); - } - let pkg_loader = wasm_pkg_client::Client::new(client_config); - - let mut releases = pkg_loader.list_all_versions(package).await.map_err(|e| { - if matches!(e, wasm_pkg_client::Error::NoRegistryForNamespace(_)) && registry.is_none() { - anyhow!("No default registry specified for wasm-pkg-loader. Create a default config, or set `registry` for package {package:?}") - } else { - e.into() - } - })?; - - releases.sort(); - - let release_version = releases - .iter() - .rev() - .find(|release| version.matches(&release.version) && !release.yanked) - .with_context(|| format!("No matching version found for {package} {version}",))?; - - let release = pkg_loader - .get_release(package, &release_version.version) - .await?; - - let digest = match &release.content_digest { - wasm_pkg_client::ContentDigest::Sha256 { hex } => format!("sha256:{hex}"), - }; - - let path = if let Ok(cached_path) = self.cache.wasm_file(&digest) { - cached_path - } else { - let mut stm = pkg_loader.stream_content(package, &release).await?; - - self.cache.ensure_dirs().await?; - let dest = self.cache.wasm_path(&digest); - - let mut file = tokio::fs::File::create(&dest).await?; - while let Some(block) = stm.next().await { - let bytes = block.context("Failed to get content from registry")?; - file.write_all(&bytes) - .await - .context("Failed to save registry content to cache")?; - } - - dest - }; - - file_content_ref(path) - } - // Copy content(s) from the given `mount` async fn copy_file_mounts( &self, @@ -812,6 +655,217 @@ fn locked_trigger(trigger_type: String, trigger: v2::Trigger) -> Result, +} + +impl WasmLoader { + /// Create a new instance of WasmLoader. + pub async fn new( + app_root: PathBuf, + cache_root: Option, + file_loading_permits: Option>, + ) -> Result { + let file_loading_permits = file_loading_permits.unwrap_or_else(|| { + std::sync::Arc::new(Semaphore::new(crate::MAX_FILE_LOADING_CONCURRENCY)) + }); + Ok(Self { + app_root, + cache: Cache::new(cache_root).await?, + file_loading_permits, + }) + } + + /// Load a Wasm source from the given ComponentSource and return a path + /// to a file location from where it can be read. + pub async fn load_component_source( + &self, + component_id: &str, + source: &v2::ComponentSource, + ) -> Result { + let content = match source { + v2::ComponentSource::Local(path) => self.app_root.join(path), + v2::ComponentSource::Remote { url, digest } => { + self.load_http_source(url, digest).await? + } + v2::ComponentSource::Registry { + registry, + package, + version, + } => { + let version = semver::Version::parse(version).with_context(|| format!("Component {component_id} specifies an invalid semantic version ({version:?}) for its package version"))?; + let version_req = format!("={version}").parse().expect("version"); + + self.load_registry_source(registry.as_ref(), package, &version_req) + .await? + } + }; + Ok(content) + } + + // Load a Wasm source from the given HTTP ContentRef source URL and + // return a ContentRef an absolute path to the local copy. + async fn load_http_source(&self, url: &str, digest: &str) -> Result { + ensure!( + digest.starts_with("sha256:"), + "invalid `digest` {digest:?}; must start with 'sha256:'" + ); + let path = if let Ok(cached_path) = self.cache.wasm_file(digest) { + cached_path + } else { + let _loading_permit = self.file_loading_permits.acquire().await?; + + self.cache.ensure_dirs().await?; + let dest = self.cache.wasm_path(digest); + verified_download( + url, + digest, + &dest, + crate::http::DestinationConvention::ContentIndexed, + ) + .await + .with_context(|| format!("Error fetching source URL {url:?}"))?; + dest + }; + Ok(path) + } + + async fn load_registry_source( + &self, + registry: Option<&wasm_pkg_client::Registry>, + package: &wasm_pkg_client::PackageRef, + version: &semver::VersionReq, + ) -> Result { + let mut client_config = wasm_pkg_client::Config::global_defaults().await?; + + if let Some(registry) = registry.cloned() { + let mapping = wasm_pkg_client::RegistryMapping::Registry(registry); + client_config.set_package_registry_override(package.clone(), mapping); + } + let pkg_loader = wasm_pkg_client::Client::new(client_config); + + let mut releases = pkg_loader.list_all_versions(package).await.map_err(|e| { + if matches!(e, wasm_pkg_client::Error::NoRegistryForNamespace(_)) && registry.is_none() { + anyhow!("No default registry specified for wasm-pkg-loader. Create a default config, or set `registry` for package {package:?}") + } else { + e.into() + } + })?; + + releases.sort(); + + let release_version = releases + .iter() + .rev() + .find(|release| version.matches(&release.version) && !release.yanked) + .with_context(|| format!("No matching version found for {package} {version}",))?; + + let release = pkg_loader + .get_release(package, &release_version.version) + .await?; + + let digest = match &release.content_digest { + wasm_pkg_client::ContentDigest::Sha256 { hex } => format!("sha256:{hex}"), + }; + + let path = if let Ok(cached_path) = self.cache.wasm_file(&digest) { + cached_path + } else { + let mut stm = pkg_loader.stream_content(package, &release).await?; + + self.cache.ensure_dirs().await?; + let dest = self.cache.wasm_path(&digest); + + let mut file = tokio::fs::File::create(&dest).await?; + while let Some(block) = stm.next().await { + let bytes = block.context("Failed to get content from registry")?; + file.write_all(&bytes) + .await + .context("Failed to save registry content to cache")?; + } + + dest + }; + + Ok(path) + } + + /// Loads a dependency + pub async fn load_component_dependency( + &self, + dependency_name: &DependencyName, + dependency: &v2::ComponentDependency, + ) -> Result<(PathBuf, Option)> { + match dependency.clone() { + v2::ComponentDependency::Version(version) => { + let version = semver::VersionReq::parse(&version).with_context(|| format!("Component dependency {dependency_name:?} specifies an invalid semantic version requirement ({version:?}) for its package version"))?; + + // This `unwrap()` should be OK because we've already validated + // this form of dependency requires a package name, i.e. the + // dependency name is not a kebab id. + let package = dependency_name.package().unwrap(); + + let content = self.load_registry_source(None, package, &version).await?; + Ok((content, None)) + } + v2::ComponentDependency::Package { + version, + registry, + package, + export, + } => { + let version = semver::VersionReq::parse(&version).with_context(|| format!("Component dependency {dependency_name:?} specifies an invalid semantic version requirement ({version:?}) for its package version"))?; + + let package = match package { + Some(package) => { + package.parse().with_context(|| format!("Component dependency {dependency_name:?} specifies an invalid package name ({package:?})"))? + } + None => { + // This `unwrap()` should be OK because we've already validated + // this form of dependency requires a package name, i.e. the + // dependency name is not a kebab id. + dependency_name + .package() + .cloned() + .unwrap() + } + }; + + let registry = match registry { + Some(registry) => { + registry + .parse() + .map(Some) + .with_context(|| format!("Component dependency {dependency_name:?} specifies an invalid registry name ({registry:?})"))? + } + None => None, + }; + + let content = self + .load_registry_source(registry.as_ref(), &package, &version) + .await?; + Ok((content, export)) + } + v2::ComponentDependency::Local { path, export } => { + let content = self.app_root.join(path); + Ok((content, export)) + } + v2::ComponentDependency::HTTP { + url, + digest, + export, + } => { + let content = self.load_http_source(&url, &digest).await?; + Ok((content, export)) + } + } + } +} + fn looks_like_glob_pattern(s: impl AsRef) -> bool { let s = s.as_ref(); glob::Pattern::escape(s) != s diff --git a/crates/manifest/src/compat.rs b/crates/manifest/src/compat.rs index b16dd0bf63..4cca17cd3a 100644 --- a/crates/manifest/src/compat.rs +++ b/crates/manifest/src/compat.rs @@ -20,6 +20,7 @@ pub fn v1_to_v2_app(manifest: v1::AppManifestV1) -> Result, + /// `targets = ["spin-2.5", "fermyon-cloud", "spinkube-0.4"]` + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub targets: Vec, /// `[application.triggers.]` #[serde(rename = "trigger", default, skip_serializing_if = "Map::is_empty")] #[schemars(schema_with = "json_schema::map_of_toml_tables")] @@ -320,7 +323,7 @@ impl ComponentDependencies { } } - anyhow::bail!("{this:?} dependency conflicts with {other:?}") + Err(anyhow!("{this:?} dependency conflicts with {other:?}")) } /// Normalize version to perform a compatibility check against another version. @@ -351,6 +354,29 @@ impl ComponentDependencies { } } +/// Identifies a deployment target. +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] +#[serde(untagged, deny_unknown_fields)] +pub enum TargetEnvironmentRef { + /// Environment definition doc reference e.g. `spin-up:3.2`, `my-host`. This is looked up + /// in the default environment catalogue (registry). + DefaultRegistry(String), + /// An environment definition doc in an OCI registry other than the default + Registry { + /// Registry or prefix hosting the environment document e.g. `ghcr.io/my/environments`. + registry: String, + /// Environment definition document name e.g. `my-spin-env:1.2`. For hosted environments + /// where you always want `latest`, omit the version tag e.g. `my-host`. + id: String, + }, + /// A local environment document file. This is expected to contain a serialised + /// EnvironmentDefinition in TOML format. + File { + /// The file path of the document. + path: PathBuf, + }, +} + mod kebab_or_snake_case { use serde::{Deserialize, Serialize}; pub use spin_serde::{KebabId, SnakeId}; diff --git a/src/commands/build.rs b/src/commands/build.rs index 16270640e9..042dfbf337 100644 --- a/src/commands/build.rs +++ b/src/commands/build.rs @@ -29,6 +29,16 @@ pub struct BuildCommand { #[clap(short = 'c', long, multiple = true)] pub component_id: Vec, + /// By default, if the application manifest specifies one or more deployment targets, Spin + /// checks that all components are compatible with those deployment targets. Specify + /// this option to bypass those target checks. + #[clap( + long = "skip-target-checks", + alias = "skip-target-check", + takes_value = false + )] + skip_target_checks: bool, + /// Run the application after building. #[clap(name = BUILD_UP_OPT, short = 'u', long = "up")] pub up: bool, @@ -43,7 +53,13 @@ impl BuildCommand { spin_common::paths::find_manifest_file_path(self.app_source.as_ref())?; notify_if_nondefault_rel(&manifest_file, distance); - spin_build::build(&manifest_file, &self.component_id).await?; + spin_build::build( + &manifest_file, + &self.component_id, + self.target_checking(), + None, + ) + .await?; if self.up { let mut cmd = UpCommand::parse_from( @@ -59,4 +75,12 @@ impl BuildCommand { Ok(()) } } + + fn target_checking(&self) -> spin_build::TargetChecking { + if self.skip_target_checks { + spin_build::TargetChecking::Skip + } else { + spin_build::TargetChecking::Check + } + } } diff --git a/src/commands/registry.rs b/src/commands/registry.rs index 39a9cc4800..8370f4b1bb 100644 --- a/src/commands/registry.rs +++ b/src/commands/registry.rs @@ -58,7 +58,7 @@ pub struct Push { #[clap(long = "compose", default_value_t = true)] pub compose: bool, - /// Specifies to perform `spin build` before pushing the application. + /// Specifies to perform `spin build` (with the default options) before pushing the application. #[clap(long, takes_value = false, env = ALWAYS_BUILD_ENV)] pub build: bool, @@ -84,7 +84,7 @@ impl Push { notify_if_nondefault_rel(&app_file, distance); if self.build { - spin_build::build(&app_file, &[]).await?; + spin_build::build_default(&app_file, self.cache_dir.clone()).await?; } let annotations = if self.annotations.is_empty() { diff --git a/src/commands/up.rs b/src/commands/up.rs index 9e11fb58cf..d0d565dcf0 100644 --- a/src/commands/up.rs +++ b/src/commands/up.rs @@ -108,7 +108,7 @@ pub struct UpCommand { #[clap(long, takes_value = false)] pub direct_mounts: bool, - /// For local apps, specifies to perform `spin build` before running the application. + /// For local apps, specifies to perform `spin build` (with the default options) before running the application. /// /// This is ignored on remote applications, as they are already built. #[clap(long, takes_value = false, env = ALWAYS_BUILD_ENV)] @@ -191,7 +191,7 @@ impl UpCommand { } if self.build { - app_source.build().await?; + app_source.build(&self.cache_dir).await?; } let mut locked_app = self .load_resolved_app_source(resolved_app_source, &working_dir) diff --git a/src/commands/up/app_source.rs b/src/commands/up/app_source.rs index 29088f0ddd..dc5e0a179a 100644 --- a/src/commands/up/app_source.rs +++ b/src/commands/up/app_source.rs @@ -56,9 +56,9 @@ impl AppSource { } } - pub async fn build(&self) -> anyhow::Result<()> { + pub async fn build(&self, cache_root: &Option) -> anyhow::Result<()> { match self { - Self::File(path) => spin_build::build(path, &[]).await, + Self::File(path) => spin_build::build_default(path, cache_root.clone()).await, _ => Ok(()), } } From 49444d3fab08f35e71852ecd8ca061ecaf888971 Mon Sep 17 00:00:00 2001 From: itowlson Date: Mon, 23 Jun 2025 16:03:43 +1200 Subject: [PATCH 2/2] Validate host requirements as part of target environments Signed-off-by: itowlson --- crates/environments/src/environment.rs | 94 ++++++++++++++++++- .../src/environment/definition.rs | 34 +++++-- .../src/environment/env_loader.rs | 14 ++- crates/environments/src/lib.rs | 27 ++++++ crates/environments/src/loader.rs | 40 ++++++-- crates/loader/src/lib.rs | 1 + crates/loader/src/local.rs | 4 +- 7 files changed, 194 insertions(+), 20 deletions(-) diff --git a/crates/environments/src/environment.rs b/crates/environments/src/environment.rs index 2ca7e1e4ad..757c4f09f6 100644 --- a/crates/environments/src/environment.rs +++ b/crates/environments/src/environment.rs @@ -18,7 +18,9 @@ use definition::WorldName; pub struct TargetEnvironment { name: String, trigger_worlds: HashMap, + trigger_capabilities: HashMap>, unknown_trigger: UnknownTrigger, + unknown_capabilities: Vec, } impl TargetEnvironment { @@ -52,6 +54,13 @@ impl TargetEnvironment { .or_else(|| self.unknown_trigger.worlds()) .unwrap_or(NO_WORLDS) } + + /// Lists all host capabilities supported for the given trigger type in this environment. + pub fn capabilities(&self, trigger_type: &TriggerType) -> &[String] { + self.trigger_capabilities + .get(trigger_type) + .unwrap_or(&self.unknown_capabilities) + } } /// How a `TargetEnvironment` should validate components associated with trigger types @@ -229,7 +238,9 @@ mod test { TargetEnvironment { name: "test".to_owned(), trigger_worlds: [("s".to_owned(), candidate_worlds)].into_iter().collect(), + trigger_capabilities: Default::default(), unknown_trigger: UnknownTrigger::Deny, + unknown_capabilities: Default::default(), } } @@ -242,7 +253,9 @@ mod test { TargetEnvironment { name: "test".to_owned(), trigger_worlds: [].into_iter().collect(), + trigger_capabilities: Default::default(), unknown_trigger: UnknownTrigger::Allow(candidate_worlds), + unknown_capabilities: Default::default(), } } @@ -260,7 +273,7 @@ mod test { assert!(env.supports_trigger_type(&"s".to_owned())); assert!(!env.supports_trigger_type(&"t".to_owned())); - let component = crate::ComponentToValidate::new("scomp", "scomp.wasm", wasm); + let component = crate::ComponentToValidate::new("scomp", "scomp.wasm", wasm, vec![]); let errs = crate::validate_component_against_environments(&[env], &"s".to_owned(), &component) .await; @@ -291,7 +304,7 @@ mod test { assert!(env.supports_trigger_type(&non_existent_trigger)); - let component = crate::ComponentToValidate::new("comp", "comp.wasm", wasm); + let component = crate::ComponentToValidate::new("comp", "comp.wasm", wasm, vec![]); let errs = crate::validate_component_against_environments( &[env], &non_existent_trigger, @@ -308,6 +321,46 @@ mod test { ); } + #[tokio::test] + async fn can_validate_component_with_host_requirement() { + let wit_path = PathBuf::from(SIMPLE_WIT_DIR); + + let wit_text = tokio::fs::read_to_string(wit_path.join("world.wit")) + .await + .unwrap(); + let wasm = generate_dummy_component(&wit_text, "spin:test/simple@1.0.0"); + + let mut env = target_simple_world(&wit_path); + env.trigger_capabilities.insert( + "s".to_owned(), + vec![ + "local_spline_reticulation".to_owned(), + "nice_cup_of_tea".to_owned(), + ], + ); + + assert!(env.supports_trigger_type(&"s".to_owned())); + assert!(!env.supports_trigger_type(&"t".to_owned())); + + let component = crate::ComponentToValidate::new( + "cscomp", + "cscomp.wasm", + wasm, + vec!["nice_cup_of_tea".to_string()], + ); + let errs = + crate::validate_component_against_environments(&[env], &"s".to_owned(), &component) + .await; + assert!( + errs.is_empty(), + "{}", + errs.iter() + .map(|e| e.to_string()) + .collect::>() + .join("\n") + ); + } + #[tokio::test] async fn unavailable_import_invalidates_component() { let wit_path = PathBuf::from(SIMPLE_WIT_DIR); @@ -319,7 +372,7 @@ mod test { let env = target_simple_world(&wit_path); - let component = crate::ComponentToValidate::new("nscomp", "nscomp.wasm", wasm); + let component = crate::ComponentToValidate::new("nscomp", "nscomp.wasm", wasm, vec![]); let errs = crate::validate_component_against_environments(&[env], &"s".to_owned(), &component) .await; @@ -346,7 +399,7 @@ mod test { let env = target_simple_world(&wit_path); - let component = crate::ComponentToValidate::new("tdscomp", "tdscomp.wasm", wasm); + let component = crate::ComponentToValidate::new("tdscomp", "tdscomp.wasm", wasm, vec![]); let errs = crate::validate_component_against_environments(&[env], &"s".to_owned(), &component) .await; @@ -359,6 +412,39 @@ mod test { ); } + #[tokio::test] + async fn unsupported_host_req_invalidates_component() { + let wit_path = PathBuf::from(SIMPLE_WIT_DIR); + + let wit_text = tokio::fs::read_to_string(wit_path.join("world.wit")) + .await + .unwrap(); + let wasm = generate_dummy_component(&wit_text, "spin:test/simple@1.0.0"); + + let env = target_simple_world(&wit_path); + + assert!(env.supports_trigger_type(&"s".to_owned())); + assert!(!env.supports_trigger_type(&"t".to_owned())); + + let component = crate::ComponentToValidate::new( + "cscomp", + "cscomp.wasm", + wasm, + vec!["nice_cup_of_tea".to_string()], + ); + let errs = + crate::validate_component_against_environments(&[env], &"s".to_owned(), &component) + .await; + assert!(!errs.is_empty()); + + let err = errs[0].to_string(); + assert!( + err.contains("Component cscomp can't run in environment test"), + "unexpected error {err}" + ); + assert!(err.contains("nice_cup_of_tea"), "unexpected error {err}"); + } + fn generate_dummy_component(wit: &str, world: &str) -> Vec { let mut resolve = wit_parser::Resolve::default(); let package_id = resolve.push_str("test", wit).expect("should parse WIT"); diff --git a/crates/environments/src/environment/definition.rs b/crates/environments/src/environment/definition.rs index 1e439b14cb..2bdf4a3645 100644 --- a/crates/environments/src/environment/definition.rs +++ b/crates/environments/src/environment/definition.rs @@ -14,22 +14,44 @@ use anyhow::Context; /// ```ignore /// # spin-up.3.2.toml /// [triggers] -/// http = ["spin:up/http-trigger@3.2.0", "spin:up/http-trigger-rc20231018@3.2.0"] -/// redis = ["spin:up/redis-trigger@3.2.0"] +/// http = { worlds = ["spin:up/http-trigger@3.2.0", "spin:up/http-trigger-rc20231018@3.2.0"], capabilities = ["local_service_chaining"] } +/// redis = { worlds = ["spin:up/redis-trigger@3.2.0"] } /// ``` #[derive(Debug, serde::Deserialize)] #[serde(deny_unknown_fields)] pub struct EnvironmentDefinition { - triggers: HashMap>, - default: Option>, + triggers: HashMap, + #[serde(default)] + default: Option, +} + +/// The environment definition for a trigger, comprising the worlds which are +/// compatible with that trigger and the host capabilities which the trigger +/// supports. +#[derive(Debug, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TriggerEnvironment { + worlds: Vec, + #[serde(default)] + capabilities: Vec, +} + +impl TriggerEnvironment { + pub fn world_refs(&self) -> &[WorldRef] { + &self.worlds + } + + pub fn capabilities(&self) -> Vec { + self.capabilities.clone() + } } impl EnvironmentDefinition { - pub fn triggers(&self) -> &HashMap> { + pub fn triggers(&self) -> &HashMap { &self.triggers } - pub fn default(&self) -> Option<&Vec> { + pub fn default(&self) -> Option<&TriggerEnvironment> { self.default.as_ref() } } diff --git a/crates/environments/src/environment/env_loader.rs b/crates/environments/src/environment/env_loader.rs index 217d063521..b0b60cfa37 100644 --- a/crates/environments/src/environment/env_loader.rs +++ b/crates/environments/src/environment/env_loader.rs @@ -128,25 +128,33 @@ async fn load_environment_from_toml( let env: EnvironmentDefinition = toml::from_str(toml_text)?; let mut trigger_worlds = HashMap::new(); + let mut trigger_capabilities = HashMap::new(); // TODO: parallel all the things // TODO: this loads _all_ triggers not just the ones we need - for (trigger_type, world_refs) in env.triggers() { + for (trigger_type, trigger_env) in env.triggers() { trigger_worlds.insert( trigger_type.to_owned(), - load_worlds(world_refs, cache, lockfile).await?, + load_worlds(trigger_env.world_refs(), cache, lockfile).await?, ); + trigger_capabilities.insert(trigger_type.to_owned(), trigger_env.capabilities()); } let unknown_trigger = match env.default() { None => UnknownTrigger::Deny, - Some(world_refs) => UnknownTrigger::Allow(load_worlds(world_refs, cache, lockfile).await?), + Some(env) => UnknownTrigger::Allow(load_worlds(env.world_refs(), cache, lockfile).await?), + }; + let unknown_capabilities = match env.default() { + None => vec![], + Some(env) => env.capabilities(), }; Ok(TargetEnvironment { name: name.to_owned(), trigger_worlds, + trigger_capabilities, unknown_trigger, + unknown_capabilities, }) } diff --git a/crates/environments/src/lib.rs b/crates/environments/src/lib.rs index 5e8371b206..7b0c83f9b6 100644 --- a/crates/environments/src/lib.rs +++ b/crates/environments/src/lib.rs @@ -102,6 +102,11 @@ async fn validate_component_against_environments( { errs.push(e); } + + let host_caps = env.capabilities(trigger_type); + if let Some(e) = validate_host_reqs(env, host_caps, component).err() { + errs.push(e); + } } if errs.is_empty() { @@ -215,3 +220,25 @@ async fn validate_wasm_against_world( }, } } + +fn validate_host_reqs( + env: &TargetEnvironment, + host_caps: &[String], + component: &ComponentToValidate, +) -> anyhow::Result<()> { + let unsatisfied: Vec<_> = component + .host_requirements() + .iter() + .filter(|host_req| !satisfies(host_caps, host_req)) + .cloned() + .collect(); + if unsatisfied.is_empty() { + Ok(()) + } else { + Err(anyhow!("Component {} can't run in environment {} because it requires the feature(s) '{}' which the environment does not support", component.id(), env.name(), unsatisfied.join(", "))) + } +} + +fn satisfies(host_caps: &[String], host_req: &String) -> bool { + host_caps.contains(host_req) +} diff --git a/crates/environments/src/loader.rs b/crates/environments/src/loader.rs index dc31f600a0..d3f634f696 100644 --- a/crates/environments/src/loader.rs +++ b/crates/environments/src/loader.rs @@ -10,6 +10,7 @@ pub(crate) struct ComponentToValidate<'a> { id: &'a str, source_description: String, wasm: Vec, + host_requirements: Vec, } impl ComponentToValidate<'_> { @@ -25,12 +26,22 @@ impl ComponentToValidate<'_> { &self.wasm } + pub fn host_requirements(&self) -> &[String] { + &self.host_requirements + } + #[cfg(test)] - pub(crate) fn new(id: &'static str, description: &str, wasm: Vec) -> Self { + pub(crate) fn new( + id: &'static str, + description: &str, + wasm: Vec, + host_requirements: Vec, + ) -> Self { Self { id, source_description: description.to_owned(), wasm, + host_requirements, } } } @@ -61,10 +72,13 @@ impl ApplicationToValidate { .component .as_ref() .ok_or_else(|| anyhow!("No component specified for trigger {}", trigger.id))?; - let (id, source, dependencies) = match component_spec { - spin_manifest::schema::v2::ComponentSpec::Inline(c) => { - (trigger.id.as_str(), &c.source, &c.dependencies) - } + let (id, source, dependencies, service_chaining) = match component_spec { + spin_manifest::schema::v2::ComponentSpec::Inline(c) => ( + trigger.id.as_str(), + &c.source, + &c.dependencies, + spin_loader::requires_service_chaining(c), + ), spin_manifest::schema::v2::ComponentSpec::Reference(r) => { let id = r.as_ref(); let Some(component) = self.manifest.components.get(r) else { @@ -73,7 +87,12 @@ impl ApplicationToValidate { trigger.id ); }; - (id, &component.source, &component.dependencies) + ( + id, + &component.source, + &component.dependencies, + spin_loader::requires_service_chaining(component), + ) } }; @@ -81,6 +100,7 @@ impl ApplicationToValidate { id, source, dependencies: WrappedComponentDependencies::new(dependencies), + requires_service_chaining: service_chaining, }) } @@ -127,10 +147,17 @@ impl ApplicationToValidate { let wasm = spin_compose::compose(&loader, &component).await.with_context(|| format!("Spin needed to compose dependencies for {} as part of target checking, but composition failed", component.id))?; + let host_requirements = if component.requires_service_chaining { + vec!["local_service_chaining".to_string()] + } else { + vec![] + }; + Ok(ComponentToValidate { id: component.id, source_description: source_description(component.source), wasm, + host_requirements, }) } } @@ -139,6 +166,7 @@ struct ComponentSource<'a> { id: &'a str, source: &'a spin_manifest::schema::v2::ComponentSource, dependencies: WrappedComponentDependencies, + requires_service_chaining: bool, } struct ComponentSourceLoader<'a> { diff --git a/crates/loader/src/lib.rs b/crates/loader/src/lib.rs index 0142425bfd..31dc952fea 100644 --- a/crates/loader/src/lib.rs +++ b/crates/loader/src/lib.rs @@ -23,6 +23,7 @@ mod fs; mod http; mod local; +pub use local::requires_service_chaining; pub use local::WasmLoader; /// Maximum number of files to copy (or download) concurrently diff --git a/crates/loader/src/local.rs b/crates/loader/src/local.rs index afe796d64b..9269ff0efe 100644 --- a/crates/loader/src/local.rs +++ b/crates/loader/src/local.rs @@ -885,7 +885,9 @@ fn file_url(path: impl AsRef) -> Result { Ok(Url::from_file_path(abs_path).unwrap().to_string()) } -fn requires_service_chaining(component: &spin_manifest::schema::v2::Component) -> bool { +/// Determines if a component requires the host to support local +/// service chaining. +pub fn requires_service_chaining(component: &spin_manifest::schema::v2::Component) -> bool { component .normalized_allowed_outbound_hosts() .unwrap_or_default()