From 6e82add10575f9e6463e50f512d104773c527143 Mon Sep 17 00:00:00 2001 From: Ruediger Klaehn Date: Mon, 9 Dec 2024 14:27:32 +0200 Subject: [PATCH 1/3] remove the migration from redb v1 v2 has been out there for long enough that people should be converted by now --- Cargo.lock | 14 +- Cargo.toml | 3 +- src/store/fs.rs | 5 +- src/store/fs/migrate_redb_v1_v2.rs | 323 ----------------------------- 4 files changed, 6 insertions(+), 339 deletions(-) delete mode 100644 src/store/fs/migrate_redb_v1_v2.rs diff --git a/Cargo.lock b/Cargo.lock index 75af50fe5..32522a788 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2079,7 +2079,7 @@ dependencies = [ "postcard", "rand", "rand_core", - "redb 2.2.0", + "redb", "serde", "ssh-key", "thiserror 2.0.4", @@ -2139,8 +2139,7 @@ dependencies = [ "rand", "range-collections", "rcgen 0.12.1", - "redb 1.5.1", - "redb 2.2.0", + "redb", "ref-cast", "reflink-copy", "rustls", @@ -3649,15 +3648,6 @@ dependencies = [ "yasna", ] -[[package]] -name = "redb" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd7f82ecd6ba647a39dd1a7172b8a1cd9453c0adee6da20cb553d83a9a460fa5" -dependencies = [ - "libc", -] - [[package]] name = "redb" version = "2.2.0" diff --git a/Cargo.toml b/Cargo.toml index d92f1e042..6008245d8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -58,7 +58,6 @@ quinn = { package = "iroh-quinn", version = "0.12", features = ["ring"] } rand = "0.8" range-collections = "0.4.0" redb = { version = "2.2.0", optional = true } -redb_v1 = { package = "redb", version = "1.5.1", optional = true } ref-cast = { version = "1.0.23", optional = true } reflink-copy = { version = "0.1.8", optional = true } self_cell = "1.0.1" @@ -98,7 +97,7 @@ testdir = "0.9.1" default = ["fs-store", "net_protocol"] downloader = ["dep:parking_lot", "tokio-util/time", "dep:hashlink"] net_protocol = ["downloader", "dep:futures-util"] -fs-store = ["dep:reflink-copy", "redb", "dep:redb_v1", "dep:tempfile"] +fs-store = ["dep:reflink-copy", "redb", "dep:tempfile"] metrics = ["iroh-metrics/metrics"] redb = ["dep:redb"] cli = ["rpc", "dep:clap", "dep:indicatif", "dep:console"] diff --git a/src/store/fs.rs b/src/store/fs.rs index f0d666f12..2ed33366f 100644 --- a/src/store/fs.rs +++ b/src/store/fs.rs @@ -87,7 +87,6 @@ use smallvec::SmallVec; use tokio::io::AsyncWriteExt; use tracing::trace_span; -mod migrate_redb_v1_v2; mod tables; #[doc(hidden)] pub mod test_support; @@ -1510,7 +1509,9 @@ impl Actor { let db = match redb::Database::create(path) { Ok(db) => db, Err(DatabaseError::UpgradeRequired(1)) => { - migrate_redb_v1_v2::run(path).map_err(ActorError::Migration)? + return Err(ActorError::Migration(anyhow::anyhow!( + "migration from v1 no longer supported" + ))) } Err(err) => return Err(err.into()), }; diff --git a/src/store/fs/migrate_redb_v1_v2.rs b/src/store/fs/migrate_redb_v1_v2.rs deleted file mode 100644 index d9ff3b07a..000000000 --- a/src/store/fs/migrate_redb_v1_v2.rs +++ /dev/null @@ -1,323 +0,0 @@ -use std::path::{Path, PathBuf}; - -use anyhow::Result; -use redb_v1::ReadableTable; -use tempfile::NamedTempFile; -use tracing::info; - -pub fn run(source: impl AsRef) -> Result { - let source = source.as_ref(); - let dir = source.parent().expect("database is not in root"); - // create the new database in a tempfile in the same directory as the old db - let target = NamedTempFile::with_prefix_in("blobs.db.migrate", dir)?; - let target = target.into_temp_path(); - info!("migrate {} to {}", source.display(), target.display()); - let old_db = redb_v1::Database::open(source)?; - let new_db = redb::Database::create(&target)?; - - let rtx = old_db.begin_read()?; - let wtx = new_db.begin_write()?; - - { - let old_blobs = rtx.open_table(old::BLOBS_TABLE)?; - let mut new_blobs = wtx.open_table(new::BLOBS_TABLE)?; - let len = old_blobs.len()?; - info!("migrate blobs table ({len} rows)"); - for (i, entry) in old_blobs.iter()?.enumerate() { - let (key, value) = entry?; - let key: crate::Hash = key.value().into(); - let value = value.value(); - if i > 0 && i % 1000 == 0 { - info!(" row {i:>6} of {len}"); - } - new_blobs.insert(key, value)?; - } - info!("migrate blobs table done"); - let old_tags = rtx.open_table(old::TAGS_TABLE)?; - let mut new_tags = wtx.open_table(new::TAGS_TABLE)?; - let len = old_tags.len()?; - info!("migrate tags table ({len} rows)"); - for (i, entry) in old_tags.iter()?.enumerate() { - let (key, value) = entry?; - let key = key.value(); - let value: crate::HashAndFormat = value.value().into(); - if i > 0 && i % 1000 == 0 { - info!(" row {i:>6} of {len}"); - } - new_tags.insert(key, value)?; - } - info!("migrate tags table done"); - let old_inline_data = rtx.open_table(old::INLINE_DATA_TABLE)?; - let mut new_inline_data = wtx.open_table(new::INLINE_DATA_TABLE)?; - let len = old_inline_data.len()?; - info!("migrate inline data table ({len} rows)"); - for (i, entry) in old_inline_data.iter()?.enumerate() { - let (key, value) = entry?; - let key: crate::Hash = key.value().into(); - let value = value.value(); - if i > 0 && i % 1000 == 0 { - info!(" row {i:>6} of {len}"); - } - new_inline_data.insert(key, value)?; - } - info!("migrate inline data table done"); - let old_inline_outboard = rtx.open_table(old::INLINE_OUTBOARD_TABLE)?; - let mut new_inline_outboard = wtx.open_table(new::INLINE_OUTBOARD_TABLE)?; - let len = old_inline_outboard.len()?; - info!("migrate inline outboard table ({len} rows)"); - for (i, entry) in old_inline_outboard.iter()?.enumerate() { - let (key, value) = entry?; - let key: crate::Hash = key.value().into(); - let value = value.value(); - if i > 0 && i % 1000 == 0 { - info!(" row {i:>6} of {len}"); - } - new_inline_outboard.insert(key, value)?; - } - info!("migrate inline outboard table done"); - } - - wtx.commit()?; - drop(rtx); - drop(old_db); - drop(new_db); - - let backup_path: PathBuf = { - let mut p = source.to_owned().into_os_string(); - p.push(".backup-redb-v1"); - p.into() - }; - info!("rename {} to {}", source.display(), backup_path.display()); - std::fs::rename(source, &backup_path)?; - info!("rename {} to {}", target.display(), source.display()); - target.persist_noclobber(source)?; - info!("opening migrated database from {}", source.display()); - let db = redb::Database::open(source)?; - Ok(db) -} - -mod new { - pub(super) use super::super::tables::*; -} - -mod old { - use bytes::Bytes; - use iroh_base::hash::BlobFormat; - use postcard::experimental::max_size::MaxSize; - use redb_v1::{RedbKey, RedbValue, TableDefinition, TypeName}; - use serde::{Deserialize, Deserializer, Serialize, Serializer}; - use smallvec::SmallVec; - - use super::super::EntryState; - use crate::util::Tag; - - pub const BLOBS_TABLE: TableDefinition = TableDefinition::new("blobs-0"); - - pub const TAGS_TABLE: TableDefinition = TableDefinition::new("tags-0"); - - pub const INLINE_DATA_TABLE: TableDefinition = - TableDefinition::new("inline-data-0"); - - pub const INLINE_OUTBOARD_TABLE: TableDefinition = - TableDefinition::new("inline-outboard-0"); - - impl redb_v1::RedbValue for EntryState { - type SelfType<'a> = EntryState; - - type AsBytes<'a> = SmallVec<[u8; 128]>; - - fn fixed_width() -> Option { - None - } - - fn from_bytes<'a>(data: &'a [u8]) -> Self::SelfType<'a> - where - Self: 'a, - { - postcard::from_bytes(data).unwrap() - } - - fn as_bytes<'a, 'b: 'a>(value: &'a Self::SelfType<'b>) -> Self::AsBytes<'a> - where - Self: 'a, - Self: 'b, - { - postcard::to_extend(value, SmallVec::new()).unwrap() - } - - fn type_name() -> TypeName { - TypeName::new("EntryState") - } - } - - impl RedbValue for HashAndFormat { - type SelfType<'a> = Self; - - type AsBytes<'a> = [u8; Self::POSTCARD_MAX_SIZE]; - - fn fixed_width() -> Option { - Some(Self::POSTCARD_MAX_SIZE) - } - - fn from_bytes<'a>(data: &'a [u8]) -> Self::SelfType<'a> - where - Self: 'a, - { - let t: &'a [u8; Self::POSTCARD_MAX_SIZE] = data.try_into().unwrap(); - postcard::from_bytes(t.as_slice()).unwrap() - } - - fn as_bytes<'a, 'b: 'a>(value: &'a Self::SelfType<'b>) -> Self::AsBytes<'a> - where - Self: 'a, - Self: 'b, - { - let mut res = [0u8; 33]; - postcard::to_slice(&value, &mut res).unwrap(); - res - } - - fn type_name() -> TypeName { - TypeName::new("iroh_base::HashAndFormat") - } - } - - impl RedbValue for Tag { - type SelfType<'a> = Self; - - type AsBytes<'a> = bytes::Bytes; - - fn fixed_width() -> Option { - None - } - - fn from_bytes<'a>(data: &'a [u8]) -> Self::SelfType<'a> - where - Self: 'a, - { - Self(Bytes::copy_from_slice(data)) - } - - fn as_bytes<'a, 'b: 'a>(value: &'a Self::SelfType<'b>) -> Self::AsBytes<'a> - where - Self: 'a, - Self: 'b, - { - value.0.clone() - } - - fn type_name() -> TypeName { - TypeName::new("Tag") - } - } - - impl RedbKey for Tag { - fn compare(data1: &[u8], data2: &[u8]) -> std::cmp::Ordering { - data1.cmp(data2) - } - } - - #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] - pub struct Hash([u8; 32]); - - impl Serialize for Hash { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - self.0.serialize(serializer) - } - } - - impl<'de> Deserialize<'de> for Hash { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let data: [u8; 32] = Deserialize::deserialize(deserializer)?; - Ok(Self(data)) - } - } - - impl MaxSize for Hash { - const POSTCARD_MAX_SIZE: usize = 32; - } - - impl From for crate::Hash { - fn from(value: Hash) -> Self { - value.0.into() - } - } - - impl RedbValue for Hash { - type SelfType<'a> = Self; - - type AsBytes<'a> = &'a [u8; 32]; - - fn fixed_width() -> Option { - Some(32) - } - - fn from_bytes<'a>(data: &'a [u8]) -> Self::SelfType<'a> - where - Self: 'a, - { - let contents: &'a [u8; 32] = data.try_into().unwrap(); - Hash(*contents) - } - - fn as_bytes<'a, 'b: 'a>(value: &'a Self::SelfType<'b>) -> Self::AsBytes<'a> - where - Self: 'a, - Self: 'b, - { - &value.0 - } - - fn type_name() -> TypeName { - TypeName::new("iroh_base::Hash") - } - } - - impl RedbKey for Hash { - fn compare(data1: &[u8], data2: &[u8]) -> std::cmp::Ordering { - data1.cmp(data2) - } - } - - /// A hash and format pair - #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, MaxSize)] - pub struct HashAndFormat { - /// The hash - pub hash: Hash, - /// The format - pub format: BlobFormat, - } - - impl From for crate::HashAndFormat { - fn from(value: HashAndFormat) -> Self { - crate::HashAndFormat { - hash: value.hash.into(), - format: value.format, - } - } - } - impl Serialize for HashAndFormat { - fn serialize(&self, serializer: S) -> std::result::Result - where - S: Serializer, - { - (self.hash, self.format).serialize(serializer) - } - } - - impl<'de> Deserialize<'de> for HashAndFormat { - fn deserialize(deserializer: D) -> std::result::Result - where - D: Deserializer<'de>, - { - let (hash, format) = <(Hash, BlobFormat)>::deserialize(deserializer)?; - Ok(Self { hash, format }) - } - } -} From 081c9d3c5faf236d146985e4c667c08ec5ee5336 Mon Sep 17 00:00:00 2001 From: Ruediger Klaehn Date: Mon, 9 Dec 2024 14:34:12 +0200 Subject: [PATCH 2/3] remove unused ref-cast dep --- Cargo.lock | 1 - Cargo.toml | 2 -- 2 files changed, 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 32522a788..1d00701e8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2140,7 +2140,6 @@ dependencies = [ "range-collections", "rcgen 0.12.1", "redb", - "ref-cast", "reflink-copy", "rustls", "self_cell", diff --git a/Cargo.toml b/Cargo.toml index 6008245d8..8a9c6f9e8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -58,7 +58,6 @@ quinn = { package = "iroh-quinn", version = "0.12", features = ["ring"] } rand = "0.8" range-collections = "0.4.0" redb = { version = "2.2.0", optional = true } -ref-cast = { version = "1.0.23", optional = true } reflink-copy = { version = "0.1.8", optional = true } self_cell = "1.0.1" serde = { version = "1", features = ["derive"] } @@ -107,7 +106,6 @@ rpc = [ "dep:nested_enum_utils", "dep:strum", "dep:futures-util", - "dep:ref-cast", "dep:portable-atomic", "dep:walkdir", "downloader", From a871549bf0c3cab2122e275c46b8eea3f168ab62 Mon Sep 17 00:00:00 2001 From: Ruediger Klaehn Date: Tue, 10 Dec 2024 10:30:28 +0200 Subject: [PATCH 3/3] add deny to be able to continue --- deny.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/deny.toml b/deny.toml index c903f9260..a909fb4a7 100644 --- a/deny.toml +++ b/deny.toml @@ -34,6 +34,7 @@ license-files = [ ignore = [ "RUSTSEC-2024-0370", # unmaintained, no upgrade available "RUSTSEC-2024-0384", # unmaintained, no upgrade available + "RUSTSEC-2024-0421", # we store dns packets by key, and don't do anything else with them. Todo: remove ] [sources]