feat: Add retry mechanism on locked/busy mbtiles files

Cargo.toml

@@ -45,6 +45,7 @@ actix-web-static-files = "4"
 anyhow = "1.0"
 approx = "0.5.1"
 async-trait = "0.1"
+backon = "1.6.0"
 base64 = "0.22.1"
 bit-set = "0.8"
 brotli = ">=5, <9"

martin-core/Cargo.toml

@@ -57,7 +57,7 @@ sprites = [
     "dep:dashmap",
 ]
 styles = ["tokio/fs", "dep:dashmap"]
-mbtiles = ["dep:mbtiles", "_tiles"]
+mbtiles = ["dep:backon", "dep:mbtiles", "dep:tokio", "_tiles"]
 pmtiles = ["dep:pmtiles", "dep:object_store", "_tiles"]
 _tiles = ["dep:base64"]
 test-pg = ["postgres"]
@@ -69,6 +69,7 @@ maplibre_native = { workspace = true, optional = true }
 [dependencies]
 actix-web = { workspace = true, optional = true }
 async-trait.workspace = true
+backon = { workspace = true, optional = true }
 base64  = { workspace = true, optional = true }
 bit-set = { workspace = true, optional = true }
 dashmap = { workspace = true, optional = true }

martin-core/src/tiles/mbtiles/source.rs

@@ -4,12 +4,15 @@ use std::fmt::{Debug, Formatter};
 use std::io;
 use std::path::PathBuf;
 use std::sync::Arc;
+use std::time::Duration;
 
 use async_trait::async_trait;
+use backon::{FibonacciBuilder, Retryable as _};
 use martin_tile_utils::{TileCoord, TileData, TileInfo};
+use mbtiles::sqlx::error::DatabaseError;
 use mbtiles::{MbtError, MbtilesPool};
 use tilejson::TileJSON;
-use tracing::trace;
+use tracing::{trace, warn};
 
 use crate::tiles::mbtiles::MbtilesError;
 use crate::tiles::{BoxedSource, MartinCoreResult, Source, UrlQuery};
@@ -33,6 +36,19 @@ impl Debug for MbtSource {
     }
 }
 
+// SQLITE_BUSY (code: 5)
+// https://sqlite.org/rescode.html#busy
+fn is_sqlite_busy(err: &MbtError) -> bool {
+    matches!(
+        err,
+        MbtError::SqlxError(se)
+            if se
+                .as_database_error()
+                .and_then(DatabaseError::code)
+                .is_some_and(|code| code == "5")
+    )
+}
+
 impl MbtSource {
     /// Creates a new `MBTiles` source from the given file path.
     pub async fn new(id: String, path: PathBuf) -> Result<Self, MbtilesError> {
@@ -41,12 +57,29 @@ impl MbtSource {
             .map_err(|e| io::Error::other(format!("{e:?}: Cannot open file {}", path.display())))
             .map_err(|e| MbtilesError::IoError(e, path.clone()))?;
 
-        let meta = mbt
-            .get_metadata()
+        // Attempt to fetch metadata with Fibonacci backoff & jitter
+        // Attempt to fetch metadata
+        let start_delay = Duration::from_millis(50);
+        let max_attempts = 10; // from 50ms to 2.75s
+        let meta = (|| async { mbt.get_metadata().await })
+            .retry(
+                FibonacciBuilder::default()
+                    .with_min_delay(start_delay)
+                    .with_max_times(max_attempts)
+                    .with_jitter(),
+            )
+            .sleep(tokio::time::sleep)
+            .when(is_sqlite_busy)
+            .notify(|_err, dur| {
+                warn!(
+                    "Database file {:?} locked (SQLITE_BUSY). Retrying in {:.2}s...",
+                    path.display(),
+                    dur.as_secs_f64()
+                );
+            })
             .await
             .map_err(|e| MbtilesError::InvalidMetadata(e.to_string(), path.clone()))?;
 
-        // Empty mbtiles should cause an error
         let tile_info = mbt
             .detect_format(&meta.tilejson)
             .await