Block Database

cache/lru/cache.go

@@ -20,6 +20,17 @@ type Cache[K comparable, V any] struct {
 	lock     sync.Mutex
 	elements *linked.Hashmap[K, V]
 	size     int
+
+	// onEvict is called with the key and value of an evicted entry, if set.
+	onEvict func(K, V)
+}
+
+// SetOnEvict sets a callback to be called with the key and value of an evicted entry.
+// The callback is called synchronously while holding the cache lock.
+func (c *Cache[K, V]) SetOnEvict(cb func(K, V)) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	c.onEvict = cb
 }
 
 func NewCache[K comparable, V any](size int) *Cache[K, V] {
@@ -34,8 +45,11 @@ func (c *Cache[K, V]) Put(key K, value V) {
 	defer c.lock.Unlock()
 
 	if c.elements.Len() == c.size {
-		oldestKey, _, _ := c.elements.Oldest()
+		oldestKey, oldestValue, _ := c.elements.Oldest()
 		c.elements.Delete(oldestKey)
+		if c.onEvict != nil {
+			c.onEvict(oldestKey, oldestValue)
+		}
 	}
 	c.elements.Put(key, value)
 }
@@ -55,14 +69,25 @@ func (c *Cache[K, V]) Get(key K) (V, bool) {
 func (c *Cache[K, _]) Evict(key K) {
 	c.lock.Lock()
 	defer c.lock.Unlock()
-
+	value, _ := c.elements.Get(key)
 	c.elements.Delete(key)
+	if c.onEvict != nil {
+		c.onEvict(key, value)
+	}
 }
 
 func (c *Cache[_, _]) Flush() {
 	c.lock.Lock()
 	defer c.lock.Unlock()
 
+	// Call onEvict for each element before clearing
+	if c.onEvict != nil {
+		iter := c.elements.NewIterator()
+		for iter.Next() {
+			c.onEvict(iter.Key(), iter.Value())
+		}
+	}
+
 	c.elements.Clear()
 }
 

cache/lru/cache_test.go

@@ -6,6 +6,8 @@ package lru
 import (
 	"testing"
 
+	"github.com/stretchr/testify/require"
+
 	"github.com/ava-labs/avalanchego/cache/cachetest"
 	"github.com/ava-labs/avalanchego/ids"
 )
@@ -19,3 +21,17 @@ func TestCacheEviction(t *testing.T) {
 	c := NewCache[ids.ID, int64](2)
 	cachetest.Eviction(t, c)
 }
+
+func TestCacheFlushWithOnEvict(t *testing.T) {
+	c := NewCache[ids.ID, int64](2)
+
+	// Track which elements were evicted
+	evicted := make(map[ids.ID]int64)
+	c.SetOnEvict(func(key ids.ID, value int64) {
+		evicted[key] = value
+	})
+
+	cachetest.Eviction(t, c)
+	require.Zero(t, c.Len())
+	require.Len(t, evicted, 3)
+}

go.mod

@@ -17,6 +17,7 @@ require (
 	github.com/ava-labs/ledger-avalanche/go v0.0.0-20241009183145-e6f90a8a1a60
 	github.com/ava-labs/libevm v1.13.14-0.3.0.rc.1
 	github.com/btcsuite/btcd/btcutil v1.1.3
+	github.com/cespare/xxhash/v2 v2.3.0
 	github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593
 	github.com/compose-spec/compose-go v1.20.2
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0
@@ -89,7 +90,6 @@ require (
 	github.com/bits-and-blooms/bitset v1.10.0 // indirect
 	github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect
 	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
-	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/cockroachdb/errors v1.9.1 // indirect
 	github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
 	github.com/cockroachdb/redact v1.1.3 // indirect

x/blockdb/README.md

@@ -0,0 +1,196 @@
+# BlockDB
+
+BlockDB is a specialized database optimized for blockchain blocks.
+
+## Key Functionalities
+
+- **O(1) Performance**: Both reads and writes complete in constant time
+- **Parallel Operations**: Multiple threads can read and write blocks concurrently without blocking
+- **Flexible Write Ordering**: Supports out-of-order block writes for bootstrapping
+- **Configurable Durability**: Optional `syncToDisk` mode guarantees immediate recoverability
+- **Automatic Recovery**: Detects and recovers unindexed blocks after unclean shutdowns
+
+## Design
+
+BlockDB uses a single index file and multiple data files. The index file maps block heights to locations in the data files, while data files store the actual block content. Data storage can be split across multiple data files based on the maximum data file size.
+
+```
+┌─────────────────┐         ┌─────────────────┐
+│   Index File    │         │  Data File 1    │
+│   (.idx)        │         │   (.dat)        │
+├─────────────────┤         ├─────────────────┤
+│ Header          │         │ Block 0         │
+│ - Version       │  ┌─────>│ - Header        │
+│ - Min Height    │  │      │ - Data          │
+│ - Max Height    │  │      ├─────────────────┤
+│ - Data Size     │  │      │ Block 1         │
+│ - ...           │  │  ┌──>│ - Header        │
+├─────────────────┤  │  │   │ - Data          │
+│ Entry[0]        │  │  │   ├─────────────────┤
+│ - Offset ───────┼──┘  │   │     ...         │
+│ - Size          │     │   └─────────────────┘
+│ - Header Size   │     │
+├─────────────────┤     │
+│ Entry[1]        │     │
+│ - Offset ───────┼─────┘
+│ - Size          │
+│ - Header Size   │
+├─────────────────┤
+│     ...         │
+└─────────────────┘
+```
+
+### File Formats
+
+#### Index File Structure
+
+The index file consists of a fixed-size header followed by fixed-size entries:
+
+```
+Index File Header (64 bytes):
+┌────────────────────────────────┬─────────┐
+│ Field                          │ Size    │
+├────────────────────────────────┼─────────┤
+│ Version                        │ 8 bytes │
+│ Max Data File Size             │ 8 bytes │
+│ Min Block Height               │ 8 bytes │
+│ Max Contiguous Height          │ 8 bytes │
+│ Max Block Height               │ 8 bytes │
+│ Next Write Offset              │ 8 bytes │
+│ Reserved                       │ 16 bytes│
+└────────────────────────────────┴─────────┘
+
+Index Entry (16 bytes):
+┌────────────────────────────────┬─────────┐
+│ Field                          │ Size    │
+├────────────────────────────────┼─────────┤
+│ Data File Offset               │ 8 bytes │
+│ Block Data Size                │ 4 bytes │
+│ Header Size                    │ 4 bytes │
+└────────────────────────────────┴─────────┘
+```
+
+#### Data File Structure
+
+Each block in the data file is stored with a block entry header followed by the raw block data:
+
+```
+Block Entry Header (26 bytes):
+┌────────────────────────────────┬─────────┐
+│ Field                          │ Size    │
+├────────────────────────────────┼─────────┤
+│ Height                         │ 8 bytes │
+│ Size                           │ 4 bytes │
+│ Checksum                       │ 8 bytes │
+│ Header Size                    │ 4 bytes │
+│ Version                        │ 2 bytes │
+└────────────────────────────────┴─────────┘
+```
+
+### Block Overwrites
+
+BlockDB allows overwriting blocks at existing heights. When a block is overwritten, the new block is appended to the data file and the index entry is updated to point to the new location, leaving the old block data as unreferenced "dead" space. However, since blocks are immutable and rarely overwritten (e.g., during reorgs), this trade-off should have minimal impact in practice.
+
+### Fixed-Size Index Entries
+
+Each index entry is exactly 16 bytes on disk, containing the offset, size, and header size. This fixed size enables direct calculation of where each block's index entry is located, providing O(1) lookups. For blockchains with high block heights, the index remains efficient, even at height 1 billion, the index file would only be ~16GB.
+
+### Durability and Fsync Behavior
+
+BlockDB provides configurable durability through the `syncToDisk` parameter:
+
+**Data File Behavior:**
+
+- **When `syncToDisk=true`**: The data file is fsync'd after every block write, guaranteeing durability against both process failures and kernel/machine failures.
+- **When `syncToDisk=false`**: Data file writes are buffered, providing durability against process failures but not against kernel or machine failures.
+
+**Index File Behavior:**
+
+- **When `syncToDisk=true`**: The index file is fsync'd every `CheckpointInterval` blocks (when the header is written).
+- **When `syncToDisk=false`**: The index file relies on OS buffering and is not explicitly fsync'd.
+
+### Recovery Mechanism
+
+On startup, BlockDB checks for signs of an unclean shutdown by comparing the data file size on disk with the indexed data size stored in the index file header. If the data files are larger than what the index claims, it indicates that blocks were written but the index wasn't properly updated before shutdown.
+
+**Recovery Process:**
+
+1. Starts scanning from where the index left off (`NextWriteOffset`)
+2. For each unindexed block found:
+   - Validates the block entry header and checksum
+   - Writes the corresponding index entry
+3. Calculates the max contiguous height and max block height
+4. Updates the index header with the updated max contiguous height, max block height, and next write offset
+
+## Usage
+
+### Creating a Database
+
+```go
+import (
+    "errors"
+    "github.com/ava-labs/avalanchego/x/blockdb"
+)
+
+config := blockdb.DefaultConfig().
+    WithDir("/path/to/blockdb")
+db, err := blockdb.New(config, logging.NoLog{})
+if err != nil {
+    fmt.Println("Error creating database:", err)
+    return
+}
+defer db.Close()
+```
+
+### Writing and Reading Blocks
+
+```go
+// Write a block with header size
+height := uint64(100)
+blockData := []byte("header:block data")
+headerSize := uint32(7) // First 7 bytes are the header
+err := db.WriteBlock(height, blockData, headerSize)
+if err != nil {
+    fmt.Println("Error writing block:", err)
+    return
+}
+
+// Read a block
+blockData, err := db.ReadBlock(height)
+if err != nil {
+    if errors.Is(err, blockdb.ErrBlockNotFound) {
+        fmt.Println("Block doesn't exist at this height")
+        return
+    }
+    fmt.Println("Error reading block:", err)
+    return
+}
+
+// Read block components separately
+headerData, err := db.ReadHeader(height)
+if err != nil {
+    if errors.Is(err, blockdb.ErrBlockNotFound) {
+        fmt.Println("Block doesn't exist at this height")
+        return
+    }
+    fmt.Println("Error reading header:", err)
+    return
+}
+bodyData, err := db.ReadBody(height)
+if err != nil {
+    if errors.Is(err, blockdb.ErrBlockNotFound) {
+        fmt.Println("Block doesn't exist at this height")
+        return
+    }
+    fmt.Println("Error reading body:", err)
+    return
+}
+```
+
+## TODO
+
+- Implement a block cache for recently accessed blocks
+- Use a buffered pool to avoid allocations on reads and writes
+- Add metrics
+- Add performance benchmarks
+- Consider supporting missing data files (currently we error if any data files are missing)