diff --git a/crates/napi/src/next_api/project.rs b/crates/napi/src/next_api/project.rs index dd8601121d60a..bcc3527325542 100644 --- a/crates/napi/src/next_api/project.rs +++ b/crates/napi/src/next_api/project.rs @@ -35,6 +35,7 @@ use turbo_tasks::{ message_queue::{CompilationEvent, Severity, TimingEvent}, trace::TraceRawVcs, }; +use turbo_tasks_backend::db_invalidation::invalidation_reasons; use turbo_tasks_fs::{ DiskFileSystem, FileContent, FileSystem, FileSystemPath, get_relative_path_to, util::uri_from_file, @@ -571,9 +572,15 @@ pub async fn project_update( pub async fn project_invalidate_persistent_cache( #[napi(ts_arg_type = "{ __napiType: \"Project\" }")] project: External, ) -> napi::Result<()> { - tokio::task::spawn_blocking(move || project.turbo_tasks.invalidate_persistent_cache()) - .await - .context("panicked while invalidating persistent cache")??; + tokio::task::spawn_blocking(move || { + // TODO: Let the JS caller specify a reason? We need to limit the reasons to ones we know + // how to generate a message for on the Rust side of the FFI. + project + .turbo_tasks + .invalidate_persistent_cache(invalidation_reasons::USER_REQUEST) + }) + .await + .context("panicked while invalidating persistent cache")??; Ok(()) } diff --git a/crates/napi/src/next_api/utils.rs b/crates/napi/src/next_api/utils.rs index 4dd256c6d3e90..9d9bed88cd946 100644 --- a/crates/napi/src/next_api/utils.rs +++ b/crates/napi/src/next_api/utils.rs @@ -11,12 +11,14 @@ use serde::Serialize; use tokio::sync::mpsc::Receiver; use turbo_tasks::{ Effects, OperationVc, ReadRef, TaskId, TryJoinIterExt, TurboTasks, TurboTasksApi, UpdateInfo, - Vc, VcValueType, get_effects, message_queue::CompilationEvent, - task_statistics::TaskStatisticsApi, trace::TraceRawVcs, + Vc, VcValueType, get_effects, + message_queue::{CompilationEvent, Severity}, + task_statistics::TaskStatisticsApi, + trace::TraceRawVcs, }; use turbo_tasks_backend::{ - DefaultBackingStorage, GitVersionInfo, NoopBackingStorage, default_backing_storage, - noop_backing_storage, + BackingStorage, DefaultBackingStorage, GitVersionInfo, NoopBackingStorage, StartupCacheState, + db_invalidation::invalidation_reasons, default_backing_storage, noop_backing_storage, }; use turbo_tasks_fs::FileContent; use turbopack_core::{ @@ -150,17 +152,51 @@ impl NextTurboTasks { } } - pub fn invalidate_persistent_cache(&self) -> Result<()> { + pub fn invalidate_persistent_cache(&self, reason_code: &str) -> Result<()> { match self { NextTurboTasks::Memory(_) => {} - NextTurboTasks::PersistentCaching(turbo_tasks) => { - turbo_tasks.backend().invalidate_storage()? - } + NextTurboTasks::PersistentCaching(turbo_tasks) => turbo_tasks + .backend() + .backing_storage() + .invalidate(reason_code)?, } Ok(()) } } +#[derive(Serialize)] +struct StartupCacheInvalidationEvent { + reason_code: Option, +} + +impl CompilationEvent for StartupCacheInvalidationEvent { + fn type_name(&self) -> &'static str { + "StartupCacheInvalidationEvent" + } + + fn severity(&self) -> Severity { + Severity::Warning + } + + fn message(&self) -> String { + let reason_msg = match self.reason_code.as_deref() { + Some(invalidation_reasons::PANIC) => { + " because we previously detected an internal error in Turbopack" + } + Some(invalidation_reasons::USER_REQUEST) => " as the result of a user request", + _ => "", // ignore unknown reasons + }; + format!( + "Turbopack's persistent cache has been deleted{reason_msg}. Builds or page loads may \ + be slower as a result." + ) + } + + fn to_json(&self) -> String { + serde_json::to_string(self).unwrap() + } +} + pub fn create_turbo_tasks( output_path: PathBuf, persistent_caching: bool, @@ -174,24 +210,24 @@ pub fn create_turbo_tasks( dirty: option_env!("CI").is_none_or(|value| value.is_empty()) && env!("VERGEN_GIT_DIRTY") == "true", }; - NextTurboTasks::PersistentCaching(TurboTasks::new( - turbo_tasks_backend::TurboTasksBackend::new( - turbo_tasks_backend::BackendOptions { - storage_mode: Some(if std::env::var("TURBO_ENGINE_READ_ONLY").is_ok() { - turbo_tasks_backend::StorageMode::ReadOnly - } else { - turbo_tasks_backend::StorageMode::ReadWrite - }), - dependency_tracking, - ..Default::default() - }, - default_backing_storage( - &output_path.join("cache/turbopack"), - &version_info, - is_ci, - )?, - ), - )) + let (backing_storage, cache_state) = + default_backing_storage(&output_path.join("cache/turbopack"), &version_info, is_ci)?; + let tt = TurboTasks::new(turbo_tasks_backend::TurboTasksBackend::new( + turbo_tasks_backend::BackendOptions { + storage_mode: Some(if std::env::var("TURBO_ENGINE_READ_ONLY").is_ok() { + turbo_tasks_backend::StorageMode::ReadOnly + } else { + turbo_tasks_backend::StorageMode::ReadWrite + }), + dependency_tracking, + ..Default::default() + }, + backing_storage, + )); + if let StartupCacheState::Invalidated { reason_code } = cache_state { + tt.send_compilation_event(Arc::new(StartupCacheInvalidationEvent { reason_code })); + } + NextTurboTasks::PersistentCaching(tt) } else { NextTurboTasks::Memory(TurboTasks::new( turbo_tasks_backend::TurboTasksBackend::new( diff --git a/packages/next/src/build/swc/index.ts b/packages/next/src/build/swc/index.ts index 2c077207f579d..4dc32695b007e 100644 --- a/packages/next/src/build/swc/index.ts +++ b/packages/next/src/build/swc/index.ts @@ -712,13 +712,14 @@ function bindingToApi( ) } - compilationEventsSubscribe() { + compilationEventsSubscribe(eventTypes?: string[]) { return subscribe>( true, async (callback) => { binding.projectCompilationEventsSubscribe( this._nativeProject, - callback + callback, + eventTypes ) } ) diff --git a/packages/next/src/build/swc/types.ts b/packages/next/src/build/swc/types.ts index 6beab32f6a547..613b8c4e564a3 100644 --- a/packages/next/src/build/swc/types.ts +++ b/packages/next/src/build/swc/types.ts @@ -238,9 +238,9 @@ export interface Project { aggregationMs: number ): AsyncIterableIterator> - compilationEventsSubscribe(): AsyncIterableIterator< - TurbopackResult - > + compilationEventsSubscribe( + eventTypes?: string[] + ): AsyncIterableIterator> invalidatePersistentCache(): Promise diff --git a/packages/next/src/build/turbopack-build/impl.ts b/packages/next/src/build/turbopack-build/impl.ts index 17e202ac6651c..45529f6fbd66d 100644 --- a/packages/next/src/build/turbopack-build/impl.ts +++ b/packages/next/src/build/turbopack-build/impl.ts @@ -19,8 +19,8 @@ import loadConfig from '../../server/config' import { hasCustomExportOutput } from '../../export/utils' import { Telemetry } from '../../telemetry/storage' import { setGlobal } from '../../trace' -import * as Log from '../output/log' import { isCI } from '../../server/ci-info' +import { backgroundLogCompilationEvents } from '../../shared/lib/turbopack/compilation-events' export async function turbopackBuild(): Promise<{ duration: number @@ -91,32 +91,7 @@ export async function turbopackBuild(): Promise<{ } ) try { - ;(async function logCompilationEvents() { - for await (const event of project.compilationEventsSubscribe()) { - switch (event.severity) { - case 'EVENT': - Log.event(event.message) - break - case 'TRACE': - Log.trace(event.message) - break - case 'INFO': - Log.info(event.message) - break - case 'WARNING': - Log.warn(event.message) - break - case 'ERROR': - Log.error(event.message) - break - case 'FATAL': - Log.error(event.message) - break - default: - break - } - } - })() + backgroundLogCompilationEvents(project) // Write an empty file in a known location to signal this was built with Turbopack await fs.writeFile(path.join(distDir, 'turbopack'), '') diff --git a/packages/next/src/server/dev/hot-reloader-turbopack.ts b/packages/next/src/server/dev/hot-reloader-turbopack.ts index 6b8bdc6222a2a..4ac8b85b5440b 100644 --- a/packages/next/src/server/dev/hot-reloader-turbopack.ts +++ b/packages/next/src/server/dev/hot-reloader-turbopack.ts @@ -98,6 +98,7 @@ import { getDevOverlayFontMiddleware } from '../../next-devtools/server/font/get import { devIndicatorServerState } from './dev-indicator-server-state' import { getDisableDevIndicatorMiddleware } from '../../next-devtools/server/dev-indicator-middleware' import { getRestartDevServerMiddleware } from '../../next-devtools/server/restart-dev-server-middleware' +import { backgroundLogCompilationEvents } from '../../shared/lib/turbopack/compilation-events' // import { getSupportedBrowsers } from '../../build/utils' const wsServer = new ws.Server({ noServer: true }) @@ -248,6 +249,9 @@ export async function createHotReloaderTurbopack( memoryLimit: opts.nextConfig.experimental?.turbopackMemoryLimit, } ) + backgroundLogCompilationEvents(project, { + eventTypes: ['StartupCacheInvalidationEvent'], + }) setBundlerFindSourceMapImplementation( getSourceMapFromTurbopack.bind(null, project, projectPath) ) diff --git a/packages/next/src/shared/lib/turbopack/compilation-events.ts b/packages/next/src/shared/lib/turbopack/compilation-events.ts new file mode 100644 index 0000000000000..e12520d8e3d17 --- /dev/null +++ b/packages/next/src/shared/lib/turbopack/compilation-events.ts @@ -0,0 +1,44 @@ +import type { Project } from '../../../build/swc/types' +import * as Log from '../../../build/output/log' + +/** + * Subscribes to compilation events for `project` and prints them using the + * `Log` library. + * + * The `signal` argument is partially implemented. The abort may not happen until the next + * compilation event arrives. + */ +export function backgroundLogCompilationEvents( + project: Project, + { eventTypes, signal }: { eventTypes?: string[]; signal?: AbortSignal } = {} +) { + ;(async function () { + for await (const event of project.compilationEventsSubscribe(eventTypes)) { + if (signal?.aborted) { + return + } + switch (event.severity) { + case 'EVENT': + Log.event(event.message) + break + case 'TRACE': + Log.trace(event.message) + break + case 'INFO': + Log.info(event.message) + break + case 'WARNING': + Log.warn(event.message) + break + case 'ERROR': + Log.error(event.message) + break + case 'FATAL': + Log.error(event.message) + break + default: + break + } + } + })() +} diff --git a/turbopack/crates/turbo-tasks-backend/Cargo.toml b/turbopack/crates/turbo-tasks-backend/Cargo.toml index 095f689d57d46..421b4dbd6ab5e 100644 --- a/turbopack/crates/turbo-tasks-backend/Cargo.toml +++ b/turbopack/crates/turbo-tasks-backend/Cargo.toml @@ -41,6 +41,7 @@ rand = { workspace = true } rayon = { workspace = true } rustc-hash = { workspace = true } serde = { workspace = true } +serde_json = { workspace = true } serde_path_to_error = { workspace = true } smallvec = { workspace = true } tokio = { workspace = true } @@ -54,7 +55,6 @@ turbo-tasks-testing = { workspace = true } [dev-dependencies] criterion = { workspace = true, features = ["async_tokio"] } regex = { workspace = true } -serde_json = { workspace = true } tempfile = { workspace = true } rstest = { workspace = true } diff --git a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs index b2a02f08dbe27..46a91fccaee12 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backend/mod.rs @@ -210,8 +210,8 @@ impl TurboTasksBackend { ))) } - pub fn invalidate_storage(&self) -> Result<()> { - self.0.backing_storage.invalidate() + pub fn backing_storage(&self) -> &B { + &self.0.backing_storage } } diff --git a/turbopack/crates/turbo-tasks-backend/src/backing_storage.rs b/turbopack/crates/turbo-tasks-backend/src/backing_storage.rs index c94d6fd7f6c8e..cf33d978e364d 100644 --- a/turbopack/crates/turbo-tasks-backend/src/backing_storage.rs +++ b/turbopack/crates/turbo-tasks-backend/src/backing_storage.rs @@ -10,7 +10,36 @@ use crate::{ utils::chunked_vec::ChunkedVec, }; -pub trait BackingStorage: 'static + Send + Sync { +/// Represents types accepted by [`TurboTasksBackend::new`]. Typically this is the value returned by +/// [`default_backing_storage`] or [`noop_backing_storage`]. +/// +/// This trait is [sealed]. External crates are not allowed to implement it. +/// +/// [`default_backing_storage`]: crate::default_backing_storage +/// [`noop_backing_storage`]: crate::noop_backing_storage +/// [`TurboTasksBackend::new`]: crate::TurboTasksBackend::new +/// [sealed]: https://predr.ag/blog/definitive-guide-to-sealed-traits-in-rust/ +pub trait BackingStorage: BackingStorageSealed { + /// Called when the database should be invalidated upon re-initialization. + /// + /// This typically means that we'll restart the process or `turbo-tasks` soon with a fresh + /// database. If this happens, there's no point in writing anything else to disk, or flushing + /// during [`KeyValueDatabase::shutdown`]. + /// + /// This can be implemented by calling [`invalidate_db`] with + /// the database's non-versioned base path. + /// + /// [`KeyValueDatabase::shutdown`]: crate::database::key_value_database::KeyValueDatabase::shutdown + /// [`invalidate_db`]: crate::database::db_invalidation::invalidate_db + fn invalidate(&self, reason_code: &str) -> Result<()>; +} + +/// Private methods used by [`BackingStorage`]. This trait is `pub` (because of the sealed-trait +/// pattern), but should not be exported outside of the crate. +/// +/// [`BackingStorage`] is exported for documentation reasons and to expose the public +/// [`BackingStorage::invalidate`] method. +pub trait BackingStorageSealed: 'static + Send + Sync { type ReadTransaction<'l>; fn lower_read_transaction<'l: 'i + 'r, 'i: 'r, 'r>( tx: &'r Self::ReadTransaction<'l>, @@ -63,16 +92,6 @@ pub trait BackingStorage: 'static + Send + Sync { category: TaskDataCategory, ) -> Result>; - /// Called when the database should be invalidated upon re-initialization. - /// - /// This typically means that we'll restart the process or `turbo-tasks` soon with a fresh - /// database. If this happens, there's no point in writing anything else to disk, or flushing - /// during [`KeyValueDatabase::shutdown`]. - /// - /// This can be implemented by calling [`crate::database::db_invalidation::invalidate_db`] with - /// the database's non-versioned base path. - fn invalidate(&self) -> Result<()>; - fn shutdown(&self) -> Result<()> { Ok(()) } diff --git a/turbopack/crates/turbo-tasks-backend/src/database/db_invalidation.rs b/turbopack/crates/turbo-tasks-backend/src/database/db_invalidation.rs index fc18af9452994..7a801cc3e6bc5 100644 --- a/turbopack/crates/turbo-tasks-backend/src/database/db_invalidation.rs +++ b/turbopack/crates/turbo-tasks-backend/src/database/db_invalidation.rs @@ -1,14 +1,73 @@ use std::{ - fs::{self, read_dir}, - io::{self, ErrorKind}, + borrow::Cow, + fs::{self, File, read_dir}, + io::{self, BufReader, BufWriter, ErrorKind, Write}, path::Path, }; use anyhow::Context; +use serde::{Deserialize, Serialize}; const INVALIDATION_MARKER: &str = "__turbo_tasks_invalidated_db"; -/// Atomically write an invalidation marker. +const EXPLANATION: &str = "The cache database has been invalidated. The existence of this file \ + will cause the cache directory to be cleaned up the next time \ + Turbopack starts up."; +const EASTER_EGG: &str = + "you just wrote me, and this is crazy, but if you see me, delete everything maybe?"; + +/// The data written to the file at [`INVALIDATION_MARKER`]. +#[derive(Serialize, Deserialize)] +struct InvalidationFile<'a> { + #[serde(skip_deserializing)] + _explanation: Option<&'static str>, + #[serde(skip_deserializing)] + _easter_egg: Option<&'static str>, + /// See [`StartupCacheState::Invalidated::reason_code`]. + reason_code: Cow<'a, str>, +} + +/// Information about if there's was a pre-existing cache or if the cache was detected as +/// invalidated during startup. +/// +/// If the cache was invalidated, the application may choose to show a warning to the user or log it +/// to telemetry. +/// +/// This value is returned by [`crate::turbo_backing_storage`] and +/// [`crate::default_backing_storage`]. +pub enum StartupCacheState { + NoCache, + Cached, + Invalidated { + /// A short code passed to [`BackingStorage::invalidate`]. This value is + /// application-specific. + /// + /// If the value is `None` or doesn't match an expected value, the application should just + /// treat this reason as unknown. The invalidation file may have been corrupted or + /// modified by an external tool. + /// + /// See [`invalidation_reasons`] for some common reason codes. + /// + /// [`BackingStorage::invalidate`]: crate::BackingStorage::invalidate + reason_code: Option, + }, +} + +/// Common invalidation reason codes. The application or libraries it uses may choose to use these +/// reasons, or it may define it's own reasons. +pub mod invalidation_reasons { + /// This invalidation reason is used by [`crate::turbo_backing_storage`] when the database was + /// invalidated by a panic. + pub const PANIC: &str = concat!(module_path!(), "::PANIC"); + /// Indicates that the user explicitly clicked a button or ran a command that invalidates the + /// cache. + pub const USER_REQUEST: &str = concat!(module_path!(), "::USER_REQUEST"); +} + +/// Atomically create an invalidation marker. +/// +/// Makes a best-effort attempt to write `reason_code` to the file, but ignores any failure with +/// writing to the file. /// /// Because attempting to delete currently open database files could cause issues, actual deletion /// of files is deferred until the next start-up (in [`check_db_invalidation_and_cleanup`]). @@ -18,9 +77,29 @@ const INVALIDATION_MARKER: &str = "__turbo_tasks_invalidated_db"; /// /// This should be run with the base (non-versioned) path, as that likely aligns closest with user /// expectations (e.g. if they're clearing the cache for disk space reasons). -pub fn invalidate_db(base_path: &Path) -> anyhow::Result<()> { - match fs::write(base_path.join(INVALIDATION_MARKER), [0u8; 0]) { - Ok(_) => Ok(()), +/// +/// In most cases, you should prefer a higher-level API like [`crate::BackingStorage::invalidate`] +/// to this one. +pub(crate) fn invalidate_db(base_path: &Path, reason_code: &str) -> anyhow::Result<()> { + match File::create_new(base_path.join(INVALIDATION_MARKER)) { + Ok(file) => { + let mut writer = BufWriter::new(file); + // ignore errors: We've already successfully invalidated the cache just by creating the + // marker file, writing the reason_code is best-effort. + let _ = serde_json::to_writer_pretty( + &mut writer, + &InvalidationFile { + _explanation: Some(EXPLANATION), + _easter_egg: Some(EASTER_EGG), + reason_code: Cow::Borrowed(reason_code), + }, + ); + let _ = writer.flush(); + Ok(()) + } + // the database was already invalidated, avoid overwriting that reason or risking concurrent + // writes to the same file. + Err(err) if err.kind() == ErrorKind::AlreadyExists => Ok(()), // just ignore if the cache directory doesn't exist at all Err(err) if err.kind() == ErrorKind::NotFound => Ok(()), Err(err) => Err(err).context("Failed to invalidate database"), @@ -31,13 +110,37 @@ pub fn invalidate_db(base_path: &Path) -> anyhow::Result<()> { /// delete any invalidated database files. /// /// This should be run with the base (non-versioned) path. -pub fn check_db_invalidation_and_cleanup(base_path: &Path) -> anyhow::Result<()> { - if fs::exists(base_path.join(INVALIDATION_MARKER))? { - // if this cleanup fails, we might try to open an invalid database later, so it's best to - // just propagate the error here. - cleanup_db(base_path)?; - }; - Ok(()) +/// +/// In most cases, you should prefer a higher-level API like +/// [`crate::KeyValueDatabaseBackingStorage::open_versioned_on_disk`] to this one. +pub(crate) fn check_db_invalidation_and_cleanup( + base_path: &Path, +) -> anyhow::Result { + match File::open(base_path.join(INVALIDATION_MARKER)) { + Ok(file) => { + // Best-effort: Try to read the reason_code from the file, if the file format is + // corrupted (or anything else) just use `None`. + let reason_code = serde_json::from_reader::<_, InvalidationFile>(BufReader::new(file)) + .ok() + .map(|contents| contents.reason_code.into_owned()); + // `file` is dropped at this point: That's important for Windows where we can't delete + // open files. + + // if this cleanup fails, we might try to open an invalid database later, so it's best + // to just propagate the error here. + cleanup_db(base_path)?; + Ok(StartupCacheState::Invalidated { reason_code }) + } + Err(err) if err.kind() == ErrorKind::NotFound => { + if fs::exists(base_path)? { + Ok(StartupCacheState::Cached) + } else { + Ok(StartupCacheState::NoCache) + } + } + Err(err) => Err(err) + .with_context(|| format!("Failed to check for {INVALIDATION_MARKER} in {base_path:?}")), + } } /// Helper for [`check_db_invalidation_and_cleanup`]. You can call this to explicitly clean up a @@ -45,7 +148,7 @@ pub fn check_db_invalidation_and_cleanup(base_path: &Path) -> anyhow::Result<()> /// /// You should not run this if the database has not yet been invalidated, as this operation is not /// atomic and could result in a partially-deleted and corrupted database. -pub fn cleanup_db(base_path: &Path) -> anyhow::Result<()> { +pub(crate) fn cleanup_db(base_path: &Path) -> anyhow::Result<()> { cleanup_db_inner(base_path).with_context(|| { format!( "Unable to remove invalid database. If this issue persists you can work around by \ diff --git a/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs b/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs index 7eb82f8b890e8..915f3570a12b3 100644 --- a/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs +++ b/turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs @@ -21,10 +21,10 @@ use turbo_tasks::{ use crate::{ GitVersionInfo, backend::{AnyOperation, TaskDataCategory}, - backing_storage::BackingStorage, + backing_storage::{BackingStorage, BackingStorageSealed}, data::CachedDataItem, database::{ - db_invalidation::{check_db_invalidation_and_cleanup, invalidate_db}, + db_invalidation::{StartupCacheState, check_db_invalidation_and_cleanup, invalidate_db}, db_versioning::handle_db_versioning, key_value_database::{KeySpace, KeyValueDatabase}, write_batch::{ @@ -32,6 +32,7 @@ use crate::{ WriteBuffer, }, }, + db_invalidation::invalidation_reasons, utils::chunked_vec::ChunkedVec, }; @@ -120,15 +121,21 @@ pub struct KeyValueDatabaseBackingStorageInner { base_path: Option, /// Used to skip calling [`invalidate_db`] when the database has already been invalidated. invalidated: Mutex, + /// We configure a panic hook to invalidate the cache. This guard cleans up our panic hook upon + /// drop. _panic_hook_guard: Option, } pub struct KeyValueDatabaseBackingStorage { + // wrapped so that `register_panic_hook` can hold a weak reference to `inner`. inner: Arc>, } +/// A wrapper type used by [`crate::turbo_backing_storage`] and [`crate::noop_backing_storage`]. +/// +/// Wraps a low-level key-value database into a higher-level [`BackingStorage`] type. impl KeyValueDatabaseBackingStorage { - pub fn new_in_memory(database: T) -> Self { + pub(crate) fn new_in_memory(database: T) -> Self { Self { inner: Arc::new(KeyValueDatabaseBackingStorageInner { database, @@ -139,19 +146,30 @@ impl KeyValueDatabaseBackingStorage { } } - pub fn open_versioned_on_disk( + /// Handles boilerplate logic for an on-disk persisted database with versioning. + /// + /// - Creates a directory per version, with a maximum number of old versions and performs + /// automatic cleanup of old versions. + /// - Checks for a database invalidation marker file, and cleans up the database as needed. + /// - [Registers a dynamic panic hook][turbo_tasks::panic_hooks] to invalidate the database upon + /// a panic. This invalidates the database using [`invalidation_reasons::PANIC`]. + /// + /// Along with returning a [`KeyValueDatabaseBackingStorage`], this returns a + /// [`StartupCacheState`], which can be used by the application for logging information to the + /// user or telemetry about the cache. + pub(crate) fn open_versioned_on_disk( base_path: PathBuf, version_info: &GitVersionInfo, is_ci: bool, database: impl FnOnce(PathBuf) -> Result, - ) -> Result + ) -> Result<(Self, StartupCacheState)> where T: Send + Sync + 'static, { - check_db_invalidation_and_cleanup(&base_path)?; + let startup_cache_state = check_db_invalidation_and_cleanup(&base_path)?; let versioned_path = handle_db_versioning(&base_path, version_info, is_ci)?; let database = (database)(versioned_path)?; - Ok(Self { + let backing_storage = Self { inner: Arc::new_cyclic( move |weak_inner: &Weak>| { let panic_hook_guard = if should_invalidate_on_panic() { @@ -164,7 +182,7 @@ impl KeyValueDatabaseBackingStorage { // or turbo-tasks failed, and it may be hard to recover. We don't want // the cache to stick around, as that may persist bugs. Make a // best-effort attempt to invalidate the database (ignoring failures). - let _ = inner.invalidate(); + let _ = inner.invalidate(invalidation_reasons::PANIC); }))) } else { None @@ -177,7 +195,8 @@ impl KeyValueDatabaseBackingStorage { } }, ), - }) + }; + Ok((backing_storage, startup_cache_state)) } } @@ -197,7 +216,7 @@ impl KeyValueDatabaseBackingStorageInner { } } - fn invalidate(&self) -> Result<()> { + fn invalidate(&self, reason_code: &str) -> Result<()> { // `base_path` can be `None` for a `NoopKvDb` if let Some(base_path) = &self.base_path { // Invalidation could happen frequently if there's a bunch of panics. We only need to @@ -212,7 +231,7 @@ impl KeyValueDatabaseBackingStorageInner { // Invalidate first, as it's a very fast atomic operation. `prevent_writes` is allowed // to be slower (e.g. wait for a lock) and is allowed to corrupt the database with // partial writes. - invalidate_db(base_path)?; + invalidate_db(base_path, reason_code)?; self.database.prevent_writes(); // Avoid redundant invalidations from future panics *invalidated_guard = true; @@ -232,6 +251,14 @@ impl KeyValueDatabaseBackingStorageInner { impl BackingStorage for KeyValueDatabaseBackingStorage +{ + fn invalidate(&self, reason_code: &str) -> Result<()> { + self.inner.invalidate(reason_code) + } +} + +impl BackingStorageSealed + for KeyValueDatabaseBackingStorage { type ReadTransaction<'l> = T::ReadTransaction<'l>; @@ -567,10 +594,6 @@ impl BackingStorage .with_context(|| format!("Looking up data for {task_id} from database failed")) } - fn invalidate(&self) -> Result<()> { - self.inner.invalidate() - } - fn shutdown(&self) -> Result<()> { self.inner.database.shutdown() } diff --git a/turbopack/crates/turbo-tasks-backend/src/lib.rs b/turbopack/crates/turbo-tasks-backend/src/lib.rs index fc266b2a0dff8..c900c3a2b86a8 100644 --- a/turbopack/crates/turbo-tasks-backend/src/lib.rs +++ b/turbopack/crates/turbo-tasks-backend/src/lib.rs @@ -18,7 +18,10 @@ use anyhow::Result; use crate::database::{noop_kv::NoopKvDb, turbo::TurboKeyValueDatabase}; pub use crate::{ backend::{BackendOptions, StorageMode, TurboTasksBackend}, - database::db_versioning::GitVersionInfo, + backing_storage::BackingStorage, + database::{ + db_invalidation, db_invalidation::StartupCacheState, db_versioning::GitVersionInfo, + }, kv_backing_storage::KeyValueDatabaseBackingStorage, }; @@ -33,12 +36,20 @@ pub type LmdbBackingStorage = KeyValueDatabaseBackingStorage< >, >; +/// Creates an [`lmdb`]-based `BackingStorage` to be passed to [`TurboTasksBackend::new`]. +/// +/// Information about the state of the on-disk cache is returned using [`StartupCacheState`]. +/// +/// This is backend is slower than [`turbo_backing_storage`], but it's a known-good database that +/// can be used when reproducing user-reported issues to isolate bugs. +/// +/// When the `lmdb` cargo feature is enabled, [`default_backing_storage`] will return this value. #[cfg(feature = "lmdb")] pub fn lmdb_backing_storage( base_path: &Path, version_info: &GitVersionInfo, is_ci: bool, -) -> Result { +) -> Result<(LmdbBackingStorage, StartupCacheState)> { use crate::database::{ fresh_db_optimization::{FreshDbOptimization, is_fresh}, read_transaction_cache::ReadTransactionCache, @@ -62,11 +73,17 @@ pub fn lmdb_backing_storage( pub type TurboBackingStorage = KeyValueDatabaseBackingStorage; +/// Creates a `BackingStorage` to be passed to [`TurboTasksBackend::new`]. +/// +/// Information about the state of the on-disk cache is returned using [`StartupCacheState`]. +/// +/// This is the fastest most-tested implementation of `BackingStorage`, and is normally returned by +/// [`default_backing_storage`]. pub fn turbo_backing_storage( base_path: &Path, version_info: &GitVersionInfo, is_ci: bool, -) -> Result { +) -> Result<(TurboBackingStorage, StartupCacheState)> { KeyValueDatabaseBackingStorage::open_versioned_on_disk( base_path.to_owned(), version_info, @@ -77,6 +94,7 @@ pub fn turbo_backing_storage( pub type NoopBackingStorage = KeyValueDatabaseBackingStorage; +/// Creates an no-op in-memory `BackingStorage` to be passed to [`TurboTasksBackend::new`]. pub fn noop_backing_storage() -> NoopBackingStorage { KeyValueDatabaseBackingStorage::new_in_memory(NoopKvDb) } @@ -84,23 +102,22 @@ pub fn noop_backing_storage() -> NoopBackingStorage { #[cfg(feature = "lmdb")] pub type DefaultBackingStorage = LmdbBackingStorage; -#[cfg(feature = "lmdb")] -pub fn default_backing_storage( - path: &Path, - version_info: &GitVersionInfo, - is_ci: bool, -) -> Result { - lmdb_backing_storage(path, version_info, is_ci) -} - #[cfg(not(feature = "lmdb"))] pub type DefaultBackingStorage = TurboBackingStorage; -#[cfg(not(feature = "lmdb"))] +/// Calls [`turbo_backing_storage`] (recommended) or `lmdb_backing_storage`, depending on if the +/// `lmdb` cargo feature is enabled. pub fn default_backing_storage( path: &Path, version_info: &GitVersionInfo, is_ci: bool, -) -> Result { - turbo_backing_storage(path, version_info, is_ci) +) -> Result<(DefaultBackingStorage, StartupCacheState)> { + #[cfg(feature = "lmdb")] + { + lmdb_backing_storage(path, version_info, is_ci) + } + #[cfg(not(feature = "lmdb"))] + { + turbo_backing_storage(path, version_info, is_ci) + } } diff --git a/turbopack/crates/turbo-tasks-backend/tests/test_config.trs b/turbopack/crates/turbo-tasks-backend/tests/test_config.trs index 3d7906ab40da4..0d86992333cdb 100644 --- a/turbopack/crates/turbo-tasks-backend/tests/test_config.trs +++ b/turbopack/crates/turbo-tasks-backend/tests/test_config.trs @@ -17,7 +17,7 @@ dirty: false, }, false - ).unwrap() + ).unwrap().0 ) ) } diff --git a/turbopack/crates/turbo-tasks-fetch/tests/test_config.trs b/turbopack/crates/turbo-tasks-fetch/tests/test_config.trs index 3d7906ab40da4..0d86992333cdb 100644 --- a/turbopack/crates/turbo-tasks-fetch/tests/test_config.trs +++ b/turbopack/crates/turbo-tasks-fetch/tests/test_config.trs @@ -17,7 +17,7 @@ dirty: false, }, false - ).unwrap() + ).unwrap().0 ) ) } diff --git a/turbopack/crates/turbopack/tests/node-file-trace.rs b/turbopack/crates/turbopack/tests/node-file-trace.rs index 5323805b82080..72753820b679a 100644 --- a/turbopack/crates/turbopack/tests/node-file-trace.rs +++ b/turbopack/crates/turbopack/tests/node-file-trace.rs @@ -284,7 +284,8 @@ fn node_file_trace_persistent(#[case] input: CaseInput) { }, false, ) - .unwrap(), + .unwrap() + .0, )) }); }