diff --git a/core/filtermaps/chain_view.go b/core/filtermaps/chain_view.go
new file mode 100644
index 00000000000..bc0de47ded4
--- /dev/null
+++ b/core/filtermaps/chain_view.go
@@ -0,0 +1,156 @@
+// Copyright 2025 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package filtermaps
+
+import (
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/core/types"
+ "github.com/ethereum/go-ethereum/log"
+)
+
+// blockchain represents the underlying blockchain of ChainView.
+type blockchain interface {
+ GetHeader(hash common.Hash, number uint64) *types.Header
+ GetCanonicalHash(number uint64) common.Hash
+ GetReceiptsByHash(hash common.Hash) types.Receipts
+}
+
+// ChainView represents an immutable view of a chain with a block id and a set
+// of receipts associated to each block number and a block hash associated with
+// all block numbers except the head block. This is because in the future
+// ChainView might represent a view where the head block is currently being
+// created. Block id is a unique identifier that can also be calculated for the
+// head block.
+// Note that the view's head does not have to be the current canonical head
+// of the underlying blockchain, it should only possess the block headers
+// and receipts up until the expected chain view head.
+type ChainView struct {
+ chain blockchain
+ headNumber uint64
+ hashes []common.Hash // block hashes starting backwards from headNumber until first canonical hash
+}
+
+// NewChainView creates a new ChainView.
+func NewChainView(chain blockchain, number uint64, hash common.Hash) *ChainView {
+ cv := &ChainView{
+ chain: chain,
+ headNumber: number,
+ hashes: []common.Hash{hash},
+ }
+ cv.extendNonCanonical()
+ return cv
+}
+
+// getBlockHash returns the block hash belonging to the given block number.
+// Note that the hash of the head block is not returned because ChainView might
+// represent a view where the head block is currently being created.
+func (cv *ChainView) getBlockHash(number uint64) common.Hash {
+ if number >= cv.headNumber {
+ panic("invalid block number")
+ }
+ return cv.blockHash(number)
+}
+
+// getBlockId returns the unique block id belonging to the given block number.
+// Note that it is currently equal to the block hash. In the future it might
+// be a different id for future blocks if the log index root becomes part of
+// consensus and therefore rendering the index with the new head will happen
+// before the hash of that new head is available.
+func (cv *ChainView) getBlockId(number uint64) common.Hash {
+ if number > cv.headNumber {
+ panic("invalid block number")
+ }
+ return cv.blockHash(number)
+}
+
+// getReceipts returns the set of receipts belonging to the block at the given
+// block number.
+func (cv *ChainView) getReceipts(number uint64) types.Receipts {
+ if number > cv.headNumber {
+ panic("invalid block number")
+ }
+ return cv.chain.GetReceiptsByHash(cv.blockHash(number))
+}
+
+// limitedView returns a new chain view that is a truncated version of the parent view.
+func (cv *ChainView) limitedView(newHead uint64) *ChainView {
+ if newHead >= cv.headNumber {
+ return cv
+ }
+ return NewChainView(cv.chain, newHead, cv.blockHash(newHead))
+}
+
+// equalViews returns true if the two chain views are equivalent.
+func equalViews(cv1, cv2 *ChainView) bool {
+ if cv1 == nil || cv2 == nil {
+ return false
+ }
+ return cv1.headNumber == cv2.headNumber && cv1.getBlockId(cv1.headNumber) == cv2.getBlockId(cv2.headNumber)
+}
+
+// matchViews returns true if the two chain views are equivalent up until the
+// specified block number. If the specified number is higher than one of the
+// heads then false is returned.
+func matchViews(cv1, cv2 *ChainView, number uint64) bool {
+ if cv1 == nil || cv2 == nil {
+ return false
+ }
+ if cv1.headNumber < number || cv2.headNumber < number {
+ return false
+ }
+ if number == cv1.headNumber || number == cv2.headNumber {
+ return cv1.getBlockId(number) == cv2.getBlockId(number)
+ }
+ return cv1.getBlockHash(number) == cv2.getBlockHash(number)
+}
+
+// extendNonCanonical checks whether the previously known reverse list of head
+// hashes still ends with one that is canonical on the underlying blockchain.
+// If necessary then it traverses further back on the header chain and adds
+// more hashes to the list.
+func (cv *ChainView) extendNonCanonical() bool {
+ for {
+ hash, number := cv.hashes[len(cv.hashes)-1], cv.headNumber-uint64(len(cv.hashes)-1)
+ if cv.chain.GetCanonicalHash(number) == hash {
+ return true
+ }
+ if number == 0 {
+ log.Error("Unknown genesis block hash found")
+ return false
+ }
+ header := cv.chain.GetHeader(hash, number)
+ if header == nil {
+ log.Error("Header not found", "number", number, "hash", hash)
+ return false
+ }
+ cv.hashes = append(cv.hashes, header.ParentHash)
+ }
+}
+
+// blockHash returns the given block hash without doing the head number check.
+func (cv *ChainView) blockHash(number uint64) common.Hash {
+ if number+uint64(len(cv.hashes)) <= cv.headNumber {
+ hash := cv.chain.GetCanonicalHash(number)
+ if !cv.extendNonCanonical() {
+ return common.Hash{}
+ }
+ if number+uint64(len(cv.hashes)) <= cv.headNumber {
+ return hash
+ }
+ }
+ return cv.hashes[cv.headNumber-number]
+}
diff --git a/core/filtermaps/checkpoints.go b/core/filtermaps/checkpoints.go
new file mode 100644
index 00000000000..f8d6500aa76
--- /dev/null
+++ b/core/filtermaps/checkpoints.go
@@ -0,0 +1,63 @@
+// Copyright 2025 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package filtermaps
+
+import (
+ _ "embed"
+ "encoding/json"
+
+ "github.com/ethereum/go-ethereum/common"
+)
+
+// checkpointList lists checkpoints for finalized epochs of a given chain.
+// This allows the indexer to start indexing from the latest available
+// checkpoint and then index tail epochs in reverse order.
+type checkpointList []epochCheckpoint
+
+// epochCheckpoint specified the last block of the epoch and the first log
+// value index where that block starts. This allows a log value iterator to
+// be initialized at the epoch boundary.
+type epochCheckpoint struct {
+ BlockNumber uint64 // block that generated the last log value of the given epoch
+ BlockId common.Hash
+ FirstIndex uint64 // first log value index of the given block
+}
+
+//go:embed checkpoints_mainnet.json
+var checkpointsMainnetJSON []byte
+
+//go:embed checkpoints_sepolia.json
+var checkpointsSepoliaJSON []byte
+
+//go:embed checkpoints_holesky.json
+var checkpointsHoleskyJSON []byte
+
+// checkpoints lists sets of checkpoints for multiple chains. The matching
+// checkpoint set is autodetected by the indexer once the canonical chain is
+// known.
+var checkpoints = []checkpointList{
+ decodeCheckpoints(checkpointsMainnetJSON),
+ decodeCheckpoints(checkpointsSepoliaJSON),
+ decodeCheckpoints(checkpointsHoleskyJSON),
+}
+
+func decodeCheckpoints(encoded []byte) (result checkpointList) {
+ if err := json.Unmarshal(encoded, &result); err != nil {
+ panic(err)
+ }
+ return
+}
diff --git a/core/filtermaps/checkpoints_holesky.json b/core/filtermaps/checkpoints_holesky.json
new file mode 100644
index 00000000000..c30856db4f2
--- /dev/null
+++ b/core/filtermaps/checkpoints_holesky.json
@@ -0,0 +1,20 @@
+[
+{"blockNumber": 814411, "blockId": "0xf763e96fc3920359c5f706803024b78e83796a3a8563bb5a83c3ddd7cbfde287", "firstIndex": 67107637},
+{"blockNumber": 914278, "blockId": "0x0678cf8d53c0d6d27896df657d98cc73bc63ca468b6295068003938ef9b0f927", "firstIndex": 134217671},
+{"blockNumber": 1048874, "blockId": "0x3620c3d52a40ff4d9fc58c3104cfa2f327f55592caf6a2394c207a5e00b4f740", "firstIndex": 201326382},
+{"blockNumber": 1144441, "blockId": "0x438fb42850f5a0d8e1666de598a4d0106b62da0f7448c62fe029b8cbad35d08d", "firstIndex": 268435440},
+{"blockNumber": 1230411, "blockId": "0xf0ee07e60a93910723b259473a253dd9cf674e8b78c4f153b32ad7032efffeeb", "firstIndex": 335543079},
+{"blockNumber": 1309112, "blockId": "0xc1646e5ef4b4343880a85b1a4111e3321d609a1225e9cebbe10d1c7abf99e58d", "firstIndex": 402653100},
+{"blockNumber": 1380522, "blockId": "0x1617cae91989d97ac6335c4217aa6cc7f7f4c2837e20b3b5211d98d6f9e97e44", "firstIndex": 469761917},
+{"blockNumber": 1476962, "blockId": "0xd978455d2618d093dfc685d7f43f61be6dae0fa8a9cb915ae459aa6e0a5525f0", "firstIndex": 536870773},
+{"blockNumber": 1533518, "blockId": "0xe7d39d71bd9d5f1f3157c35e0329531a7950a19e3042407e38948b89b5384f78", "firstIndex": 603979664},
+{"blockNumber": 1613787, "blockId": "0xa793168d135c075732a618ec367faaed5f359ffa81898c73cb4ec54ec2caa696", "firstIndex": 671088003},
+{"blockNumber": 1719099, "blockId": "0xc4394c71a8a24efe64c5ff2afcdd1594f3708524e6084aa7dadd862bd704ab03", "firstIndex": 738196914},
+{"blockNumber": 1973165, "blockId": "0xee3a9e959a437c707a3036736ec8d42a9261ac6100972c26f65eedcde315a81d", "firstIndex": 805306333},
+{"blockNumber": 2274844, "blockId": "0x76e2d33653ed9282c63ad09d721e1f2e29064aa9c26202e20fc4cc73e8dfe5f6", "firstIndex": 872415141},
+{"blockNumber": 2530503, "blockId": "0x59f4e45345f8b8f848be5004fe75c4a28f651864256c3aa9b2da63369432b718", "firstIndex": 939523693},
+{"blockNumber": 2781903, "blockId": "0xc981e91c6fb69c5e8146ead738fcfc561831f11d7786d39c7fa533966fc37675", "firstIndex": 1006632906},
+{"blockNumber": 3101713, "blockId": "0xc7baa577c91d8439e3fc79002d2113d07ca54a4724bf2f1f5af937b7ba8e1f32", "firstIndex": 1073741382},
+{"blockNumber": 3221770, "blockId": "0xa6b8240b7883fcc71aa5001b5ba66c889975c5217e14c16edebdd6f6e23a9424", "firstIndex": 1140850360}
+]
+
diff --git a/core/filtermaps/checkpoints_mainnet.json b/core/filtermaps/checkpoints_mainnet.json
new file mode 100644
index 00000000000..d2b031a474d
--- /dev/null
+++ b/core/filtermaps/checkpoints_mainnet.json
@@ -0,0 +1,256 @@
+[
+{"blockNumber": 4166218, "blockId": "0xdd767e0426256179125551e8e40f33565a96d1c94076c7746fa79d767ed4ad65", "firstIndex": 67108680},
+{"blockNumber": 4514014, "blockId": "0x33a0879bdabea4a7a3f2b424388cbcbf2fbd519bddadf13752a259049c78e95d", "firstIndex": 134217343},
+{"blockNumber": 4817415, "blockId": "0x4f0e8c7dd04fbe0985b9394575b19f13ea66a2a628fa5b08178ce4b138c6db80", "firstIndex": 201326352},
+{"blockNumber": 5087733, "blockId": "0xc84cd5e9cda999c919803c7a53a23bb77a18827fbde401d3463f1e9e52536424", "firstIndex": 268435343},
+{"blockNumber": 5306107, "blockId": "0x13f028b5fc055d23f55a92a2eeecfbcfbda8a08e4cd519ce451ba2e70428f5f9", "firstIndex": 335544094},
+{"blockNumber": 5509918, "blockId": "0x1ed770a58a7b4d4a828b7bb44c8820a674d562b23a6a0139981abe4c489d4dad", "firstIndex": 402652853},
+{"blockNumber": 5670390, "blockId": "0x3923ee6a62e6cc5132afdadf1851ae4e73148e6fbe0a8319cafd2a120c98efa3", "firstIndex": 469761897},
+{"blockNumber": 5826139, "blockId": "0xe61bc6ef03c333805f26319e1688f82553f98aa5e902b200e0621a3371b69050", "firstIndex": 536870853},
+{"blockNumber": 5953029, "blockId": "0x43d710b1b7243b848400975048ccefdfaba091c692c7f01c619d988886cc160f", "firstIndex": 603979580},
+{"blockNumber": 6102846, "blockId": "0xa100b2018f6545cc689656b4b846677b138955b7efd30e850cd14c246430ba18", "firstIndex": 671088291},
+{"blockNumber": 6276718, "blockId": "0xb832ac448b06c104ba50faefd58b0b94d53c0fba5cb268086adad4db99c2f35f", "firstIndex": 738197399},
+{"blockNumber": 6448696, "blockId": "0x48e8ae6f729ad6c76b6cf632bd52a6df7886ed55be09d43c5004fcc1463e533b", "firstIndex": 805305988},
+{"blockNumber": 6655974, "blockId": "0xac395971a6ffc30f807848f68b97b2834f8ea13478a7615860b6a69e3d0823ca", "firstIndex": 872415033},
+{"blockNumber": 6873949, "blockId": "0xc522ddb1113b1e9a87b2bdcb11ce78756beba6454a890122f121a032b5769354", "firstIndex": 939523784},
+{"blockNumber": 7080953, "blockId": "0x3606de577d80120d1edbb64bad7fa6795e788bae342866a98cc58ce2f7575045", "firstIndex": 1006632796},
+{"blockNumber": 7267002, "blockId": "0xad770882a69d216e955e34fef84851e56c0de82deacd6187a7a41f6170cd6c6d", "firstIndex": 1073741045},
+{"blockNumber": 7466708, "blockId": "0x17a48817b3a65aba333a5b56f3ff2e86fbcc19e184b046a5305a5182fdd8eb8a", "firstIndex": 1140850680},
+{"blockNumber": 7661807, "blockId": "0xa74731ee775fbd3f4d9313c68562737dd7c8d2c9eb968791d8abe167e16ddc96", "firstIndex": 1207959112},
+{"blockNumber": 7834556, "blockId": "0xe4b4812448075508cb05a0e3257f91b49509dc78cd963676a633864db6e78956", "firstIndex": 1275068095},
+{"blockNumber": 7990068, "blockId": "0x07bd4ca38abb4584a6209e04035646aa545ebbb6c948d438d4c25bfd9cb205fa", "firstIndex": 1342176620},
+{"blockNumber": 8143032, "blockId": "0x0e3149e9637290b044ee693b8fcb66e23d22db3ad0bdda32962138ba18e59f3f", "firstIndex": 1409285949},
+{"blockNumber": 8297660, "blockId": "0x34cd24f80247f7dfaf316b2e637f4b62f72ecc90703014fb25cb98ad044fc2c0", "firstIndex": 1476394911},
+{"blockNumber": 8465137, "blockId": "0x4452fa296498248d7f10c9dc6ec1e4ae7503aa07f491e6d38b21aea5d2c658a8", "firstIndex": 1543503744},
+{"blockNumber": 8655820, "blockId": "0x7bdb9008b30be420f7152cc294ac6e5328eed5b4abd954a34105de3da24f3cc6", "firstIndex": 1610612619},
+{"blockNumber": 8807187, "blockId": "0xde03e3bfddc722c019f0b59bc55efabcd5ab68c6711f4c08d0390a56f396590d", "firstIndex": 1677721589},
+{"blockNumber": 8911171, "blockId": "0xe44f342de74ab05a2a994f8841bdf88f720b9dc260177ba4030d0f7077901324", "firstIndex": 1744830310},
+{"blockNumber": 8960320, "blockId": "0x79764f9ff6e0fe4848eda1805687872021076e4e603112861af84181395ac559", "firstIndex": 1811938893},
+{"blockNumber": 9085994, "blockId": "0x24a101d1c8a63367a0953d10dc79c3b587a93bd7fd382084708adefce0b8363f", "firstIndex": 1879047965},
+{"blockNumber": 9230924, "blockId": "0xb176a98d3acd855cbb75265fb6be955a8d51abc771e021e13275d5b3ecb07eeb", "firstIndex": 1946156668},
+{"blockNumber": 9390535, "blockId": "0x640f5e2d511a5141878d57ae7a619f19b72a2bd3ef019cf0a22d74d93d9acf07", "firstIndex": 2013265733},
+{"blockNumber": 9515674, "blockId": "0xff4a7b6b21aeaeb6e1a75ecd22b1f34c058a0ce1477ce90a8ce78165fd1d0941", "firstIndex": 2080374553},
+{"blockNumber": 9659426, "blockId": "0xc351455249343b41e9171e183612b68c3c895271c62bd2c53d9e3ab1aa865aa1", "firstIndex": 2147483567},
+{"blockNumber": 9794018, "blockId": "0xde98035b4b7f9449c256239b65c7ff2c0330de44dee190106d0a96fb6f683238", "firstIndex": 2214592213},
+{"blockNumber": 9923840, "blockId": "0x881da313a1e2b6fab58a1d6fa65b5dacfdc9d68a3112a647104955b5233f84e3", "firstIndex": 2281701302},
+{"blockNumber": 10042435, "blockId": "0x451f6459640a6f54e2a535cc3a49cfc469861da3ddc101840ab3aef9e17fa424", "firstIndex": 2348810174},
+{"blockNumber": 10168883, "blockId": "0x5d16ff5adf0df1e4dc810da60af37399ef733be7870f21112b8c2cfff4995dd9", "firstIndex": 2415918783},
+{"blockNumber": 10289554, "blockId": "0x85d5690f15a787c43b9a49e8dd6e324f0b3e0c9796d07c0cfb128e5c168f5488", "firstIndex": 2483027930},
+{"blockNumber": 10386676, "blockId": "0x20f675ea72db448024a8a0b8e3ec180cac37a5910575bc32f8d9f5cdfe3c2649", "firstIndex": 2550136212},
+{"blockNumber": 10479675, "blockId": "0x014abb07acf2330cc78800ca1f564928f2daccca4b389bf5c59f4b840d843ec0", "firstIndex": 2617245218},
+{"blockNumber": 10562661, "blockId": "0xd437607a3f81ce8b7c605e167ce5e52bf8a3e02cdc646997bd0ccc57a50ad7d1", "firstIndex": 2684354520},
+{"blockNumber": 10641508, "blockId": "0x2e8ab6470c29f90ac23dcfc58310f0208f5d0e752a0c7982a77a223eca104082", "firstIndex": 2751462730},
+{"blockNumber": 10717156, "blockId": "0x8820447b6429dd12be603c1c130be532e9db065bb4bc6b2a9d4551794d63789a", "firstIndex": 2818571831},
+{"blockNumber": 10784549, "blockId": "0xc557daab80a7cdc963d62aa881faf3ab1baceff8e027046bcd203e432e0983b3", "firstIndex": 2885680800},
+{"blockNumber": 10848651, "blockId": "0xede1b0de5db6685a6f589096ceb8fccb08d3ff60e8b00a93caa4a775b48e07fc", "firstIndex": 2952789740},
+{"blockNumber": 10909166, "blockId": "0x989db675899d13323006a4d6174557e3c5501c672afd60d8bd902fc98d37e92e", "firstIndex": 3019897599},
+{"blockNumber": 10972902, "blockId": "0x5484050cc2c7d774bc5cd6af1c2ef8c19d1de12dabe25867c9b365924ea10434", "firstIndex": 3087007422},
+{"blockNumber": 11036597, "blockId": "0x1e3686e19056587c385262d5b0a07b3ec04e804c2d59e9aaca1e5876e78f69ae", "firstIndex": 3154116231},
+{"blockNumber": 11102520, "blockId": "0x339cf302fe813cce3bb9318b860dfa8be7f688413f38a6ea1987a1b84d742b4b", "firstIndex": 3221224863},
+{"blockNumber": 11168162, "blockId": "0xc0fa21ea090627610bcac4732dff702633f310cabafc42bc500d3d4805198fe0", "firstIndex": 3288334273},
+{"blockNumber": 11233707, "blockId": "0x491c37a479b8cf22eaa3654ae34c5ddc4627df8c58ca8a6979159e1710428576", "firstIndex": 3355442691},
+{"blockNumber": 11300526, "blockId": "0xb7366d2a24df99002cffe0c9a00959c93ef0dcfc3fd17389e2020bf5caa788eb", "firstIndex": 3422551480},
+{"blockNumber": 11367621, "blockId": "0xce53df5080c5b5238bb7717dfbfd88c2f574cfbb3d91f92b57171a00e9776cd2", "firstIndex": 3489660710},
+{"blockNumber": 11431881, "blockId": "0x2a08ff9c4f6fd152166213d902f0870822429f01d5f90e384ac54a3eac0ceb3a", "firstIndex": 3556768626},
+{"blockNumber": 11497107, "blockId": "0x1f99c6b65f2b1cb06ed1786c6a0274ff1b9eacab6cb729fcd386f10ebbd88123", "firstIndex": 3623878389},
+{"blockNumber": 11560104, "blockId": "0xebe6924817bbdfe52af49667da1376bae5a2994b375d4b996e8ff2683744e37a", "firstIndex": 3690986640},
+{"blockNumber": 11625129, "blockId": "0xbe6eee325329ee2fe632d8576864c29dd1c79bab891dc0a22d5b2ac87618d26e", "firstIndex": 3758095773},
+{"blockNumber": 11690397, "blockId": "0xc28bf55f858ddf5b82d1ceb3b5258b90a9ca34df8863a1c652c4d359f5748fdf", "firstIndex": 3825204492},
+{"blockNumber": 11755087, "blockId": "0x0c10cde6ce1bbe24dc57347fe4aaebc17b7d8e8d7d97e3db573133477f494740", "firstIndex": 3892314051},
+{"blockNumber": 11819674, "blockId": "0x36b694a1776c94e4c6ae4a410931b2086de47a83e437517040e3290ce9afff67", "firstIndex": 3959422445},
+{"blockNumber": 11883358, "blockId": "0x21f447aca9ddf94ed71df9fa3648a12acc2ba603f89f24c4784936864c41945f", "firstIndex": 4026531743},
+{"blockNumber": 11948524, "blockId": "0x71a52b6cce80d3a552b0daa18beb952facf81a89bc7ca769d08ac297f317507a", "firstIndex": 4093640009},
+{"blockNumber": 12013168, "blockId": "0x9a7fb369b8d8cd0edd0d890d636096f20c63abb7eb5798ad1e578cac599e3db8", "firstIndex": 4160748475},
+{"blockNumber": 12078711, "blockId": "0x5de09329413b0c2f58d926f225197552a335ba3d5544d7bdb45e7574f78c9b8d", "firstIndex": 4227858275},
+{"blockNumber": 12143640, "blockId": "0xbeafc0e1e0586f5a95f00f2a796d7df122c79c187aa2d917129297f24b8306bd", "firstIndex": 4294967145},
+{"blockNumber": 12208005, "blockId": "0x052487095cdd4a604808e6c14e30fb68b3fa546d35585b315f287219d38ef77c", "firstIndex": 4362075289},
+{"blockNumber": 12272465, "blockId": "0x82c8a50413bd67a0d6f53b085adcd9ae8c25ecc07ed766fa80297a8dcae63b29", "firstIndex": 4429184610},
+{"blockNumber": 12329418, "blockId": "0x294c147e48d32c217ff3f27a3c8c989f15eee57a911408ec4c28d4f13a36bb3b", "firstIndex": 4496292968},
+{"blockNumber": 12382388, "blockId": "0x8c2555965ff735690d2d94ececc48df4700e079c7b21b8e601a30d4e99bc4b5b", "firstIndex": 4563401809},
+{"blockNumber": 12437052, "blockId": "0x2e38362031f36a0f3394da619dcc03be03c19700594cbd1df84c2c476a87de63", "firstIndex": 4630511012},
+{"blockNumber": 12490026, "blockId": "0x122749c02a55c9c2a1e69068f54b6c1d25419eb743e3553aba91acf1daeadc35", "firstIndex": 4697619920},
+{"blockNumber": 12541747, "blockId": "0xfb9f12aa2902da798ac05fab425434f8c7ce98050d67d416dbb32f98c21f66f7", "firstIndex": 4764728267},
+{"blockNumber": 12597413, "blockId": "0x9a7a399c2904ac8d0fec580550525e7e1a73d8f65f739bf7c05d86e389d0d3f7", "firstIndex": 4831837757},
+{"blockNumber": 12651950, "blockId": "0xb78dcb572cdafb9c4e2f3863ef518a3b2df0cd4f76faa26a423b2ca0c1cde734", "firstIndex": 4898946491},
+{"blockNumber": 12706472, "blockId": "0xfd21f41ec6b0c39287d7d48c134d1212a261c53d65db99739994b003150bbad1", "firstIndex": 4966054796},
+{"blockNumber": 12762929, "blockId": "0xc94d994bc40b2ae7dc23cf2b92cc01e84915f090bb57c0d9a67584bd564d3916", "firstIndex": 5033164307},
+{"blockNumber": 12816689, "blockId": "0x7770c72f22cbf6ccf7ab85d203088f7ede89632cf0042c690102f926a90bd09d", "firstIndex": 5100273412},
+{"blockNumber": 12872408, "blockId": "0x2e008b8c952d828875d777f7912f472af96ffc977f2ceae884006682cab6b8ed", "firstIndex": 5167381625},
+{"blockNumber": 12929718, "blockId": "0x85eb0ed3c5910c6a01b65ef0a5b76c59c2cdb5094e6e27eb87c751d77bcc2c88", "firstIndex": 5234491305},
+{"blockNumber": 12988757, "blockId": "0xdf12045bea73af18d4e71f8be8e334160f78b85f96a3535a4056409d8b61355a", "firstIndex": 5301600237},
+{"blockNumber": 13049172, "blockId": "0xf07608d97a101cd9a95fee9d9062a15bcb333263e555f8cfa31da037e0468f30", "firstIndex": 5368709080},
+{"blockNumber": 13108936, "blockId": "0x42739341db582d2f39b91ec9e8cc758777ca3f6ff9f25cd98883619fd5f026a7", "firstIndex": 5435817013},
+{"blockNumber": 13175495, "blockId": "0x564f25eacb229350b7c648b5828169e7a0344ae62e866206828e2cfad8947f10", "firstIndex": 5502926476},
+{"blockNumber": 13237721, "blockId": "0x0973425abec0fa6319701b46e07c2373b0580e3adbed6900aad27d5bf26dcb95", "firstIndex": 5570035419},
+{"blockNumber": 13298771, "blockId": "0xf3a16fec5be808c9f7782fb578dc8cef7f8e2110f7289bd03c0cc13977dd1518", "firstIndex": 5637143840},
+{"blockNumber": 13361281, "blockId": "0x3c0b6364201ca9221b61af3de27a3a87e111870b8c7efc43a6d8496e98c68690", "firstIndex": 5704253046},
+{"blockNumber": 13421819, "blockId": "0x2f472e57997b95558b99e3e5e7e0e8d4dbf8b71c081aac6536c9ff5925dac2ce", "firstIndex": 5771361231},
+{"blockNumber": 13480620, "blockId": "0xc4d689e87464a0c83c661c8e3a0614c370631de857f7e385b161dfe8bacd3e71", "firstIndex": 5838469468},
+{"blockNumber": 13535793, "blockId": "0xe7674bacc8edce9fb3efd59b92c97da48fe7ace1de314b4a67d7d032fc3bb680", "firstIndex": 5905578026},
+{"blockNumber": 13590588, "blockId": "0x6a3e86bdce7dd7d8792e1af9156edd8c3ffee7c20fed97001f58a9a2699f6594", "firstIndex": 5972687757},
+{"blockNumber": 13646707, "blockId": "0xab404a5d3709cf571b04e9493f37116eeb5dd2bc9dc10c48387c1e0199013d69", "firstIndex": 6039797165},
+{"blockNumber": 13703025, "blockId": "0x20e2fde15b8fe56f5dd7ab0f324c552038167ed44864bf3978e531ae68d6d138", "firstIndex": 6106905803},
+{"blockNumber": 13761024, "blockId": "0x2ae49275e13e780f1d29aea8507b2a708ff7bfe977efac93e050273b8b3a8164", "firstIndex": 6174015107},
+{"blockNumber": 13819468, "blockId": "0xb9d19cb31dedb1128b11cad9ffd6e58c70fe7ba65ba68f1ac63668ac5160ad85", "firstIndex": 6241124350},
+{"blockNumber": 13877932, "blockId": "0x80b1ff0bb069a8479360a15eaa84ba30da02cfacadc564837f4b1c90478addb8", "firstIndex": 6308232256},
+{"blockNumber": 13935384, "blockId": "0xe1f5469a559a6114dd469af61b118b9d9551a69bbd49a4e88f2a2d724830c871", "firstIndex": 6375341632},
+{"blockNumber": 13994042, "blockId": "0x25188fb75f2328c870ade7c38ef42ff5fddef9c4e364eebe4c5d8d9cc3ecabab", "firstIndex": 6442449799},
+{"blockNumber": 14051123, "blockId": "0xf4ef2bce9ee9222bdcf6b3a0c204676d9345e211e10c983e523930274e041ef1", "firstIndex": 6509559107},
+{"blockNumber": 14109189, "blockId": "0x80b730c28f75d8cb5ec2fb736341cd87cb4ecb2c9c614e0a4ecc0f9812675d50", "firstIndex": 6576667347},
+{"blockNumber": 14166822, "blockId": "0xf662a24b91684fa8ac462b31071f406de8d6183dba46d30d690f4407bc6af36f", "firstIndex": 6643777079},
+{"blockNumber": 14222488, "blockId": "0x7333e324c96b12f11a38d1fc2ddb4860e018b90f5dc10f3dbe19f7679bb95535", "firstIndex": 6710885890},
+{"blockNumber": 14277180, "blockId": "0x4373c1000e8e10179657689e2f0e42f88bd1601ecb4a5d83970d10287f6654cc", "firstIndex": 6777994595},
+{"blockNumber": 14331080, "blockId": "0x9c708a750a3f284ec0ee950110b36fd488cb1ec24cd0c2ea72c19551ec5c42a5", "firstIndex": 6845103719},
+{"blockNumber": 14384243, "blockId": "0x34ce7503b76335aa18dec880b0cefd388a29e0fcff6f2e1ddda8fb8c0ac1daf0", "firstIndex": 6912212376},
+{"blockNumber": 14437670, "blockId": "0x79842efd3e406b41f51935fe2e6ad20a7dd5a9db2280ebd7f602ed93da1e3c24", "firstIndex": 6979320543},
+{"blockNumber": 14489204, "blockId": "0xcd12addf0afdc229e9fe3bd0a34677a3826c5e78d4baf715f8ed36b736d6627a", "firstIndex": 7046430591},
+{"blockNumber": 14541688, "blockId": "0x55f617abf208a73fc467e8cb5feead586b671dbb0f6281570b3c44b8eabb2b9e", "firstIndex": 7113538755},
+{"blockNumber": 14594551, "blockId": "0xc7211bf772e93c8c2f945fcb6098b47c3455604cb8b94a505cb5cb720914c369", "firstIndex": 7180646025},
+{"blockNumber": 14645065, "blockId": "0x6d5b0326f4b22e2b0196986a514f23ec6e9a62f70f53300a22b21ff661a6ef7e", "firstIndex": 7247756883},
+{"blockNumber": 14695926, "blockId": "0x0a77272250e43b4bb46c02eb76944881a3c6b00a21bb9086a8229199bd62d97a", "firstIndex": 7314865843},
+{"blockNumber": 14746330, "blockId": "0xd677fdbaf8efb1bfdc138ac6b2bd5d0e890a29acb1f52f40169181ad517b0d31", "firstIndex": 7381974956},
+{"blockNumber": 14798546, "blockId": "0xbb277e8623acd2ce2340cf32f6c0ddab70fd95d862287f68a3c37250a70619cd", "firstIndex": 7449082890},
+{"blockNumber": 14848230, "blockId": "0x587b39f11bdaa2091291c7c3947e88df2e91e7997f2375dfd43b6e310a538582", "firstIndex": 7516192636},
+{"blockNumber": 14897646, "blockId": "0xf5b5c9d0c024ca0c0f0c6171871f609687f4ccb064ededbd61176cf23a9011e8", "firstIndex": 7583299602},
+{"blockNumber": 14950782, "blockId": "0x50549486afaf92a4c3520012b325e914ef77a82e4d6530a71f9b1cca31bfae18", "firstIndex": 7650409868},
+{"blockNumber": 15004101, "blockId": "0x7edac55dea3ee4308db60b9bc0524836226fe301e085b3ce39105bd145ba7fc3", "firstIndex": 7717517503},
+{"blockNumber": 15056903, "blockId": "0xb4cfd02d435718598179cdba3f5c11eb8653fe97ec8d89c60673e3e07b8dfc94", "firstIndex": 7784627997},
+{"blockNumber": 15108302, "blockId": "0x53c77a7de4515e9e93467a76f04cc401834bcdd64e9dfa03cf6d2844a6930293", "firstIndex": 7851736988},
+{"blockNumber": 15159526, "blockId": "0x1a31ad84b423254d7ff24e7eca54048ed8cc13cec5eb7289bf3f98ed4de9f724", "firstIndex": 7918844431},
+{"blockNumber": 15211013, "blockId": "0xe5d491e1d6cc5322454143b915c106be1bf28114a41b054ba5e5cfe0abecafba", "firstIndex": 7985953942},
+{"blockNumber": 15264389, "blockId": "0xd9939bb9e58e95d2672c1148b4ec5730204527d3f3fc98ca03a67dc85cf3d710", "firstIndex": 8053063187},
+{"blockNumber": 15315862, "blockId": "0x7254f99c4bb05235d5b437984c9132164e33182d4ce11a3847999da5c28b4092", "firstIndex": 8120172147},
+{"blockNumber": 15364726, "blockId": "0x11b57547579d9009679e327f57e308fe86856391805bc3c86e7b39daae890f52", "firstIndex": 8187281042},
+{"blockNumber": 15412886, "blockId": "0xbe3602b1dbef9015a3ec7968ac7652edf4424934b6bf7b713b99d8556f1d9444", "firstIndex": 8254390023},
+{"blockNumber": 15462792, "blockId": "0x3348ca4e14ac8d3c6ac6df676deaf3e3b5e0a11b599f73bd9739b74ebd693efe", "firstIndex": 8321499024},
+{"blockNumber": 15509914, "blockId": "0xbc98fd6b71438d5a169f9373172fea799fa3d22a8e6fe648d35e1070f2261113", "firstIndex": 8388606521},
+{"blockNumber": 15558748, "blockId": "0x5fa2cf499276ae74a5b8618990e71ed11a063619afe25c01b46e6252eba14c19", "firstIndex": 8455716577},
+{"blockNumber": 15604217, "blockId": "0x78a608e13d2eb3c5fed81a19b829ede88071cf01ea9ff58112a7472435f97c30", "firstIndex": 8522825668},
+{"blockNumber": 15651869, "blockId": "0xd465d861d925d1475440782ff16c2b3361ba3c8e169d7cc90eb8dfc0f31b0aac", "firstIndex": 8589934080},
+{"blockNumber": 15700968, "blockId": "0x71e3def131271e02c06ca945d14a995703a48faac1334a9e2e2321edd0b504d0", "firstIndex": 8657043390},
+{"blockNumber": 15762986, "blockId": "0x9b1b51dca2eae29162ca66968a77b45175f134b44aea3defadcb924f83e0b944", "firstIndex": 8724151376},
+{"blockNumber": 15814455, "blockId": "0x3c04a509cb6304d3df4bef57e0119d9e615ab737ec0b4a7deada6e5f57d9f873", "firstIndex": 8791260562},
+{"blockNumber": 15865639, "blockId": "0x9e9e26148c774518ecf362c0e7c65a5c1b054a8a3e4e36036c70e273fac6147c", "firstIndex": 8858368894},
+{"blockNumber": 15920564, "blockId": "0x9efe1d4dbfd9aa891ac0cffd3e1422a27ba2ea4add211b6900a2242cdb0f0ca0", "firstIndex": 8925477950},
+{"blockNumber": 15974371, "blockId": "0xc63ccef7bc35a0b431a411f99fe581b322d00cfc6422d078696808a5658a32ac", "firstIndex": 8992587107},
+{"blockNumber": 16032913, "blockId": "0x3e60957224964669a8646914e3166553b9f4256d5be160b17995d838af3ef137", "firstIndex": 9059696632},
+{"blockNumber": 16091057, "blockId": "0x12b346047bb49063ab6d9e737775924cf05c52114202ddb1a2bdaf9caabbfe0c", "firstIndex": 9126804912},
+{"blockNumber": 16150977, "blockId": "0x49318a32ff0ce979c4061c1c34db2a94fb06e7669c93742b75aff14a134fa598", "firstIndex": 9193913896},
+{"blockNumber": 16207432, "blockId": "0xf7870865edf81be4389a0be01468da959de703df0d431610814d16ed480176e4", "firstIndex": 9261019778},
+{"blockNumber": 16262582, "blockId": "0x25818e0f4d54af6c44ef7b23add34409a47de3ab1c905889478f3ec8ad173ec3", "firstIndex": 9328131320},
+{"blockNumber": 16319695, "blockId": "0x25de4b1c18cc503f5d12b4fa9072d33a11fa503a3dbeb9ab3d016b57c1e5cd4d", "firstIndex": 9395240790},
+{"blockNumber": 16373605, "blockId": "0x3794a5e0d2aa10baf1e6a5ec623d6089fdd39799eff633017d8df5144526939f", "firstIndex": 9462349509},
+{"blockNumber": 16423494, "blockId": "0xe0217d947ba3865dfc9288e0c890b0996457bb9d18467bd125e86bbb0052b57f", "firstIndex": 9529458033},
+{"blockNumber": 16474853, "blockId": "0xd454f033d190f22f9e56f0209ea1eeb3b6257805d5d88650d2759eb4d24821b7", "firstIndex": 9596567055},
+{"blockNumber": 16525689, "blockId": "0x8a23cbbf3e258e13f5a1ada434366796cb4a3e5b1062455582fb2bc3ab991541", "firstIndex": 9663674943},
+{"blockNumber": 16574203, "blockId": "0xc1a5b7d26e8222bd2d56ef4108f75d69f7c116707d348950834e00962241a4f8", "firstIndex": 9730785112},
+{"blockNumber": 16622622, "blockId": "0x3ddb3ef7a4309bd788258fb0d62613c89a0b4de715f4e12f6017a194d19d6481", "firstIndex": 9797893665},
+{"blockNumber": 16672585, "blockId": "0x8aa5e9f72b261f9e2a9eb768483d1bbd84d3a88fdb1346f6a9a7f262fd28ba41", "firstIndex": 9865002893},
+{"blockNumber": 16720124, "blockId": "0x2128f8baf264166e37554d5c31a06de58d9ccfb663117358251da548a23a060f", "firstIndex": 9932111275},
+{"blockNumber": 16769162, "blockId": "0x6b3e849482d3222032740ad6b8f98e24636c82682a6a3572b1ef76dfebc66821", "firstIndex": 9999217824},
+{"blockNumber": 16818311, "blockId": "0xe45f57381978a2bfc85bd20af1c41e2b630412642ac4f606b477f05f030ef5d9", "firstIndex": 10066328668},
+{"blockNumber": 16869531, "blockId": "0xa154555266d24dc1f4885af5fafcf8cab3de788998cf69e1d28f56aa13a40c43", "firstIndex": 10133437302},
+{"blockNumber": 16921611, "blockId": "0xf1f829b4ab5eec6e243916dd530993fa11eef5510fd730e8d09ead6b380355a1", "firstIndex": 10200547185},
+{"blockNumber": 16974870, "blockId": "0x1a33202b95926ae4cb8e6e99d8d150f3c50d817b3a316452bdf428c971dabde5", "firstIndex": 10267655914},
+{"blockNumber": 17031277, "blockId": "0x706c9dd0dc81e7ac29d2ea0f826e6b8a1dcb5adb1b904ff6e43260729c9fd0a7", "firstIndex": 10334764934},
+{"blockNumber": 17086330, "blockId": "0x085a80cafe96b520105b9a1f8e7a2bbc9474da24da7e6344ca7c4d32db822f92", "firstIndex": 10401871892},
+{"blockNumber": 17141311, "blockId": "0x33ec6513dfa515bc5f6356476b4eb075a8064181d6aaf6aa1a1e18887e342f74", "firstIndex": 10468982364},
+{"blockNumber": 17190907, "blockId": "0x6f41273d3bf30d3347e7eb68872a49b3ac947f314543478be7a28a55e5c41a3c", "firstIndex": 10536090817},
+{"blockNumber": 17237199, "blockId": "0x9a87a14a128c0345a366940f821a14f16719de628658ac0628e410a72d723e90", "firstIndex": 10603200178},
+{"blockNumber": 17287181, "blockId": "0x9c6e78adcf562ac63c103e3e5a02f025023079aca79bdd6ef18f7bd2a6271c29", "firstIndex": 10670309183},
+{"blockNumber": 17338652, "blockId": "0x1b747da97b2397a293602af57514dab4ca1010bb6c601ff05cb2012dd1124ebb", "firstIndex": 10737418023},
+{"blockNumber": 17389337, "blockId": "0xbc3c0ca1e5989605b9b59c94b418562eb17ccbce30e45ac8531cf0b3867a6b2c", "firstIndex": 10804522857},
+{"blockNumber": 17442261, "blockId": "0x1ec341be1cbd09f559bfa3d3e39a341d8e21052eeb7880931d43d086651733b7", "firstIndex": 10871635535},
+{"blockNumber": 17497787, "blockId": "0x6069880d486f2548599df1e14e12752d3eb9bc99843a98cd6631c22be1b58554", "firstIndex": 10938744657},
+{"blockNumber": 17554322, "blockId": "0x69b2564bc00b1f310f6b416912869d7530d7864bf7d70d55c7ace554f129b989", "firstIndex": 11005852829},
+{"blockNumber": 17608492, "blockId": "0x7d590653d5fa52c0d3ee453a77d2088504f57adcef35cd57c567afb554608457", "firstIndex": 11072961972},
+{"blockNumber": 17664272, "blockId": "0xdc16159d3500cdc7410873102f41fc55de2a8a41e3779c4b70e6224a541e2b9e", "firstIndex": 11140070967},
+{"blockNumber": 17715101, "blockId": "0x655e33c4e81182464ea0b0e1fdbc53ce53902431db5107326b816091a4564652", "firstIndex": 11207179487},
+{"blockNumber": 17764042, "blockId": "0x54439184f31cd83ba06b48b6dbfdd744ae7246355be1327b44744058711d05c0", "firstIndex": 11274287303},
+{"blockNumber": 17814383, "blockId": "0xfb453bc951360c76fb09bb1b9a3e39d23ececa0adb93368cc3f41f0457845089", "firstIndex": 11341397984},
+{"blockNumber": 17864648, "blockId": "0x32a68823ef4ec0cbab2fe50c97e3f462b575e8b117da40d00c710b4c66ee1d6d", "firstIndex": 11408505657},
+{"blockNumber": 17913366, "blockId": "0x04b944aab8a4ff91b77c2191817cf051766100c227616a3746af53407e740124", "firstIndex": 11475614351},
+{"blockNumber": 17961690, "blockId": "0x08bee7cc0b764106ca01dd5370b617879487ffb423688c96e948dce125990f45", "firstIndex": 11542723488},
+{"blockNumber": 18011048, "blockId": "0x94c39d3a64f3e9a91b1d98554cd29e1390e30fa61cfa4e909c503eee2fd9f165", "firstIndex": 11609833142},
+{"blockNumber": 18061209, "blockId": "0x2ee9ade68955c030488c8a30537bdf948355f7dd5ae64942b5bfce1be6650e19", "firstIndex": 11676941316},
+{"blockNumber": 18111692, "blockId": "0xd6c4fd0c1cc20ed5e7960bb5043e9e5e9c66a4d2ec5709ac9797fff678435640", "firstIndex": 11744050346},
+{"blockNumber": 18166212, "blockId": "0x3262588c2ef79a3b3f6a3db6435202d22f5667cd48c136b0797404901525c9ff", "firstIndex": 11811159686},
+{"blockNumber": 18218743, "blockId": "0x935bd9a4164ff7ecd09a37b916ce5bf78487bd19377b5b17be153e39318aee74", "firstIndex": 11878268593},
+{"blockNumber": 18271236, "blockId": "0xe58ebb821f27e3665898f390802a3d129d217b3a3ee36d890a85cf22a0a8aa33", "firstIndex": 11945376750},
+{"blockNumber": 18323007, "blockId": "0x3997a841468efa1bc614bfc3de4502274901b04b428f87a1f3086dfd78cda1eb", "firstIndex": 12012485748},
+{"blockNumber": 18372443, "blockId": "0xc44a13a5d02e8dc39f355de5e21ce7bb311ce7f4d9114ff480dce235a169e416", "firstIndex": 12079595370},
+{"blockNumber": 18421829, "blockId": "0x7da63a0b613d8745597b2ac64fd5cc8b2fb14b24d163b12a0a39d7d3d4ff7b5c", "firstIndex": 12146703582},
+{"blockNumber": 18471706, "blockId": "0xd632a1893f415ff618f4b612a7687e6af1f12feeed81f46f0022090829c1eb4c", "firstIndex": 12213812677},
+{"blockNumber": 18522301, "blockId": "0x44fa2cf08145ae40e8e42f4e6b4ab7df360a17c5a065ce45fcc41b51bee011f4", "firstIndex": 12280921639},
+{"blockNumber": 18572935, "blockId": "0x72b8ab4c78c90425ee054b4806a8be703da0febdf1d51866358ec2bd21ba9529", "firstIndex": 12348029751},
+{"blockNumber": 18623431, "blockId": "0x8c4cb2f13501d9788820280c6f16692d0737258c3896f1e4bded32d838febf7f", "firstIndex": 12415138965},
+{"blockNumber": 18675470, "blockId": "0x523b73c19ea8b3ae32ef141a83ef9855e667ebf51443cfcabd1a06659359062a", "firstIndex": 12482247454},
+{"blockNumber": 18725728, "blockId": "0x0cfbd131eb5dad51488238079fba29a63eebb5c32d1a543cb072e48dc2104ef3", "firstIndex": 12549356369},
+{"blockNumber": 18778387, "blockId": "0xc4906c77af8058b9f172a4f0e8788c7887f05caa5ac752b38b5387080f74ae49", "firstIndex": 12616465992},
+{"blockNumber": 18835044, "blockId": "0x49c5e07f409a841dc81f3ef8417f1951f8fcc13c90134f9d2a0cd11938f9fa36", "firstIndex": 12683575082},
+{"blockNumber": 18883308, "blockId": "0x386a58dd5f79a419eeb05075b07b3ff3bc836a265c9688854a504223b1d6a830", "firstIndex": 12750683753},
+{"blockNumber": 18933635, "blockId": "0xd3881292147589bd2e192769e5c9175b5d03a453fe1ef3c4b5b6858ac9402a2f", "firstIndex": 12817792470},
+{"blockNumber": 18988254, "blockId": "0xcbe72dfa15428ac21b9c59c703ceaa0eb4b2205927687261d7aaed3dbb3783ea", "firstIndex": 12884882858},
+{"blockNumber": 19041325, "blockId": "0x92b077e1c2f8819da728f0307c914fdcd57eba14ea07d9a45c28d1ed8ffff576", "firstIndex": 12952010530},
+{"blockNumber": 19089163, "blockId": "0x43f8ab2d3dfc34c8e18cba903074d54e235dc546f19c4eb78245a522c266c84e", "firstIndex": 13019119228},
+{"blockNumber": 19140629, "blockId": "0xab7b7ae5424b18105a13b657fa6099d4ab67fde5baff39fe6e4de707397e995c", "firstIndex": 13086228236},
+{"blockNumber": 19192118, "blockId": "0x451327e6a5cf6ce1c8c14c01687dc5f719f3c2176f46bac4f264616256e30d1c", "firstIndex": 13153337116},
+{"blockNumber": 19237836, "blockId": "0x9b260d6be369557d1dc88aca423e2697e697d941d1b726c183015b5649e248c8", "firstIndex": 13220445421},
+{"blockNumber": 19291271, "blockId": "0x4878c28d79e1f71bc11e062eb61cb52ae6a18b670b0f9bea38b477944615078e", "firstIndex": 13287554254},
+{"blockNumber": 19344448, "blockId": "0x56243b9ad863bf90953fe9aa6e64a426629384db1190e70dce79575d30595f7e", "firstIndex": 13354663659},
+{"blockNumber": 19394948, "blockId": "0x195173b64dda7908d6aa39a63c8bdd29ec181d401e369d513be1308550d0ddcb", "firstIndex": 13421771935},
+{"blockNumber": 19443075, "blockId": "0xd39c1d60996475e65d1ab5b4e755f510ca466564a8155d35db6667988d6c0e44", "firstIndex": 13488880427},
+{"blockNumber": 19488383, "blockId": "0x28956eb8856fa8db59c02585016b8baf43bc44bc35b00bdaf8a6babe51101c5c", "firstIndex": 13555977105},
+{"blockNumber": 19534584, "blockId": "0x2421c97b0f140185d4c20943cd4ed7d7424468482feb76e3003a1cc69da3fd7b", "firstIndex": 13623097580},
+{"blockNumber": 19579602, "blockId": "0x25f96529028e9f51c59aec9ce8de282b7dd67066fd46a1694130698ed0f40d8b", "firstIndex": 13690207623},
+{"blockNumber": 19621517, "blockId": "0x4f6f6e0a0488f3d51823bc4b07c292348c259b1866968f77ee76b66b37101c75", "firstIndex": 13757315529},
+{"blockNumber": 19665085, "blockId": "0x00f9315f89681b44bff46f1bad8894bc6dfae1c459d3d6520f9881861304a496", "firstIndex": 13824425382},
+{"blockNumber": 19709229, "blockId": "0x24e022b21ae1ba8a3e8c87cb9734aa1d1810fc4a69fe147d3ebb1ff0df8bcc15", "firstIndex": 13891534799},
+{"blockNumber": 19755387, "blockId": "0x77f184b7183b1a351760d242041249464b42cfaa6fbc4326f352b06bb3b21a02", "firstIndex": 13958642483},
+{"blockNumber": 19803894, "blockId": "0xf37eb1b054a6d61272940361f386eb744cded84d15c3250a7eabadede257371c", "firstIndex": 14025751618},
+{"blockNumber": 19847885, "blockId": "0x4659649fa8a3b4bbe8978673ba9a22ae20352c7052b676d373b5a51b1967ffa4", "firstIndex": 14092848654},
+{"blockNumber": 19894193, "blockId": "0x15606bdc0f1a710bd69443c7154d4979aece9329977b65990c9b39d6df84ed5c", "firstIndex": 14159970181},
+{"blockNumber": 19938551, "blockId": "0x6a8f4571924ed902bd8e71bf8ed9cc9d72cabeabc410277c8f0fc2b477d00eb7", "firstIndex": 14227077892},
+{"blockNumber": 19985354, "blockId": "0x7b6fb6376410b4d9e5d7ee02f78b2054e005dd2976eea47fc714f66b967dc285", "firstIndex": 14294187965},
+{"blockNumber": 20028440, "blockId": "0x9b37440b71c24756b8855b8012432b84276ae94c80aa1ccc8b70a7705992103c", "firstIndex": 14361296503},
+{"blockNumber": 20071780, "blockId": "0xa2ed129f343f3d60419772ec5635edcd36b8680c9419b6626e2bc84b230c709b", "firstIndex": 14428405230},
+{"blockNumber": 20113832, "blockId": "0xe7a610e8bcbf8ded141ebc7142de03dfc54b1bcc79e3bf8d07fad4e42b665bba", "firstIndex": 14495512019},
+{"blockNumber": 20156854, "blockId": "0xbe09704f65a70ef8843d9c8e511ddc989ea139dbe94cdfe37f52b03620d62385", "firstIndex": 14562622430},
+{"blockNumber": 20200135, "blockId": "0x9a58c34d5f77342e94065d119905c000223cd988c4b11f1539fff20737159630", "firstIndex": 14629731923},
+{"blockNumber": 20244389, "blockId": "0x1e733f0db9ef21183107259b3c2408c78fa5a01469928cd295f3ea7e8eedda45", "firstIndex": 14696840011},
+{"blockNumber": 20288489, "blockId": "0xb5ad7edd86b181226c8c7be0a08069e3955234e797426843fff9de0f57ec59cc", "firstIndex": 14763949714},
+{"blockNumber": 20333582, "blockId": "0x8040c209f5cd1738ee0f85c2f1db7c43a420d148680c7390fd1701b9f0bb671a", "firstIndex": 14831058335},
+{"blockNumber": 20377087, "blockId": "0x08fdc4cd246b6ae9d4a45646b0aed6af3bb330eb6cd4c8b93646157e7b002b84", "firstIndex": 14898167722},
+{"blockNumber": 20421699, "blockId": "0x5a2912b5fc2f02df33b655155990f92dcaacda5b75427fe3d87fb38f36b1c17d", "firstIndex": 14965275691},
+{"blockNumber": 20467194, "blockId": "0x3deaf4325c461004b090b0261996c645ab529c1471feaf7dc2bbe1f128180297", "firstIndex": 15032385211},
+{"blockNumber": 20512397, "blockId": "0x37e39697ec1b7683a6202be250ffaee7a1102e8030f87550b94af05ec66cec83", "firstIndex": 15099493973},
+{"blockNumber": 20557443, "blockId": "0x8e9c04468f3111eab8b1f6a58b277862c624861c237cadecc53ec249bd811bda", "firstIndex": 15166602882},
+{"blockNumber": 20595899, "blockId": "0x9787555fe57e4650002257eb2c88f1ef435b99d406e33fe2f889be180123ef25", "firstIndex": 15233709908},
+{"blockNumber": 20638606, "blockId": "0x70681cffd159ce2e580dbbbe8fa6b5343dbcb081429cdda6c577e615bef4ef05", "firstIndex": 15300820678},
+{"blockNumber": 20683605, "blockId": "0xb32662d5e241132ffe2249caea67f5746a6f4382297b2ac87c81e2794faf1f7a", "firstIndex": 15367929350},
+{"blockNumber": 20728630, "blockId": "0x15a817c846928b673032d5eacd0cff7a04217d268457aa30a322ecca32be4d49", "firstIndex": 15435037830},
+{"blockNumber": 20771519, "blockId": "0x542bc7b9804bbc45f4be470f4dc56f215a4dec71fed71eba2ffc804afd262b95", "firstIndex": 15502145990},
+{"blockNumber": 20815097, "blockId": "0x798cdd51c964fcf18561d70095d9613b84ba836817972799c9dfd0bfbe1e042b", "firstIndex": 15569256033},
+{"blockNumber": 20857859, "blockId": "0xfb5bb066d419a651d8e0186569eb4e8d8bcd5181d8f02e0d578b5dfe2fc738dd", "firstIndex": 15636364671},
+{"blockNumber": 20896890, "blockId": "0x834b8d6fad779e4cf8214128f6c93d7387b6d6279e517f6f0a284b5d831cc3ae", "firstIndex": 15703472902},
+{"blockNumber": 20939387, "blockId": "0x7adee7c78420c711efa216c61e0b561e581d7ff0331efd91ee16a609b34cfdc2", "firstIndex": 15770582325},
+{"blockNumber": 20981303, "blockId": "0x6f5d7b0cc6dad5eb258176e07de21795a8347d68f7303f06934046e0236bea6d", "firstIndex": 15837691713},
+{"blockNumber": 21023216, "blockId": "0x96cfe35a45df1297a36f42c59ebe706ab0473dfbf59ce910b5c5a8dbf696de1c", "firstIndex": 15904799667},
+{"blockNumber": 21068378, "blockId": "0x93753875ff330d922b23f823203198f3b1bb8833367c6b6a8f896ff54be2c12d", "firstIndex": 15971909040},
+{"blockNumber": 21112445, "blockId": "0x6ac02fa6ae486b86aba562eaf6f3d883befaa8ebedcfd8d74bdb7368d42deee3", "firstIndex": 16039003625},
+{"blockNumber": 21155992, "blockId": "0x25f76896b4b693bafb79e9a535e2bf00ed62a577e35209749346e8e79a60bb71", "firstIndex": 16106126344},
+{"blockNumber": 21200962, "blockId": "0x725f2befe913cb2659d262e2d3b6f79a706b31c557d52669471da22347ec8287", "firstIndex": 16173235265},
+{"blockNumber": 21244663, "blockId": "0x6778c4194f54e70939da38853daddb22bfaf160d35617ab05d0f5c476741147b", "firstIndex": 16240344735},
+{"blockNumber": 21290273, "blockId": "0x433ac819c40bd3061205fe0ece0645eec73f54a0a5c1559c981f983345bc0154", "firstIndex": 16307453543},
+{"blockNumber": 21336156, "blockId": "0x261dc8c1639d505624150d2388d15ed10bfb4c3ce9c0c327a4ec26531689a097", "firstIndex": 16374562466},
+{"blockNumber": 21378880, "blockId": "0x5c78b2b70553140dfdfdd4f415b98f88e74f74662315834038fd99042277d917", "firstIndex": 16441671104},
+{"blockNumber": 21421613, "blockId": "0x854532f9d1c77627b763f9cbc7099a653d59554ed57fa763bc218834c82955fe", "firstIndex": 16508780351},
+{"blockNumber": 21466875, "blockId": "0xb8b83cc62084e948235ef4b5973bf7fd988fa28bcaa72f7d38ad8e50de729618", "firstIndex": 16575888599},
+{"blockNumber": 21511942, "blockId": "0xe806a28bc1b7f8cd752c8ceedbe081d49773d4558a9fb95e3357c0c07172522d", "firstIndex": 16642996907},
+{"blockNumber": 21550291, "blockId": "0x1f3e26d303e7a2a9b0614f12f62b189da365b3947c5fe2d99ed2711b37fe7daa", "firstIndex": 16710106826},
+{"blockNumber": 21592690, "blockId": "0xa1408cfbc693faee4425e8fd9e83a181be535c33f874b56c3a7a114404c4f686", "firstIndex": 16777215566},
+{"blockNumber": 21636275, "blockId": "0x704734c2d0351f8ccd38721a9a4b80c063368afaaa857518d98498180a502bba", "firstIndex": 16844323959},
+{"blockNumber": 21681066, "blockId": "0x1e738568ed393395c498b109ad61c0286747318aae0364936f19a7b6aba94aef", "firstIndex": 16911433076},
+{"blockNumber": 21725592, "blockId": "0xee87b7948e25a7498a247c616a0fbaa27f21b004e11fc56f2a20c03791ed8122", "firstIndex": 16978540993}
+]
+
diff --git a/core/filtermaps/checkpoints_sepolia.json b/core/filtermaps/checkpoints_sepolia.json
new file mode 100644
index 00000000000..3e3ea23b243
--- /dev/null
+++ b/core/filtermaps/checkpoints_sepolia.json
@@ -0,0 +1,63 @@
+[
+{"blockNumber": 3246675, "blockId": "0x36bf7de9e1f151963088ca3efa206b6e78411d699d2f64f3bf86895294275e0b", "firstIndex": 67107286},
+{"blockNumber": 3575582, "blockId": "0x08931012467636d3b67ae187790951daed2bb6423f9cd94e166df787b856788d", "firstIndex": 134217672},
+{"blockNumber": 3694264, "blockId": "0x1f35f276a3c78e5942ee285fcbd0c687691853c599a2f5b174ea88f653bc9514", "firstIndex": 201326578},
+{"blockNumber": 3725632, "blockId": "0x3bcb264c56c3eeab6c8588145f09dff3fb5f821d9fc1e7b92264b14314dae553", "firstIndex": 268433636},
+{"blockNumber": 3795390, "blockId": "0x2d1ef2815bb8e018b275fa65540b98265285016aff12596bd89a3b1442d248eb", "firstIndex": 335542953},
+{"blockNumber": 3856683, "blockId": "0x8a9a46d6f53975cd9ec829c3c307a99fb62b8428cefb63ffe06d17143649c3ee", "firstIndex": 402648835},
+{"blockNumber": 3869370, "blockId": "0x2e8c04e7e5e96d09260b65d77b1770b4105b0db2ee7d638c48f086b8afac17db", "firstIndex": 469759276},
+{"blockNumber": 3938357, "blockId": "0xf20f2cdbcc412d5340e31955d14a6526ea748ba99b5ec70b6615bdb18bcd4cfb", "firstIndex": 536868027},
+{"blockNumber": 3984894, "blockId": "0x0bcd886b3cebb884d5beeaf5ad15ee1514968b5ad07177297c7d9c00f27aa406", "firstIndex": 603968430},
+{"blockNumber": 4002664, "blockId": "0x7d3575b6ca685468fa5a5fa9ff9d5fac4415b0a67a3ed87d3530f127db32fff4", "firstIndex": 671088417},
+{"blockNumber": 4113187, "blockId": "0x3a5313ac5b602134bb73535b22801261e891ccb7bd660ab20e0a536dc46d3e13", "firstIndex": 738197016},
+{"blockNumber": 4260758, "blockId": "0xe30fb9a304d3602896a5716d310f67ba34ccef7f809a3ead4b2d991cb9ee4eb0", "firstIndex": 805306270},
+{"blockNumber": 4391131, "blockId": "0x3958478c1c3be9b7caedbcc96230ed446d711e56580e324bc2fcf903fc87c90f", "firstIndex": 872415115},
+{"blockNumber": 4515650, "blockId": "0x46a3a7b97a9dff4ef4dc2c1cc5cd501f2182d9548655b77b5e07a2dbb41071a4", "firstIndex": 939523930},
+{"blockNumber": 4634818, "blockId": "0x2197d0dd3925c1d7ba3e2c4eef20035b68efc0a2506f76ddd9e481e0ce8ca6e1", "firstIndex": 1006628557},
+{"blockNumber": 4718295, "blockId": "0xcce7bb4af1a41e6056ef68192e60c738be01ac3e071ed1ec52cead08a39995ce", "firstIndex": 1073734698},
+{"blockNumber": 4753438, "blockId": "0xa60e043728a369cdf39a399bd7a903085ee9386f38176947578e5692b4b01f65", "firstIndex": 1140843192},
+{"blockNumber": 4786522, "blockId": "0x10629cadc00e65f193fa4d10ecd2bf1855e442814c4a409d19aae9eb895dce13", "firstIndex": 1207956586},
+{"blockNumber": 4811706, "blockId": "0xf1e94111f0086733bdcb4a653486a8b94ec998b61dda0af0fd465c9b4e344f87", "firstIndex": 1275058221},
+{"blockNumber": 4841796, "blockId": "0xa530f7dd72881ac831affdc579c9d75f6d4b6853b1f1894d320bd9047df5f9eb", "firstIndex": 1342177155},
+{"blockNumber": 4914835, "blockId": "0xbd8321e354f72c4190225f8ed63d4aba794b3b568677d985e099cb62d9d36bae", "firstIndex": 1409286143},
+{"blockNumber": 4992519, "blockId": "0x4a06a5a4aa5bc52151937cc1c0f8da691a0282e94aab8b73b9faa87da8d028de", "firstIndex": 1476384367},
+{"blockNumber": 5088668, "blockId": "0xb7d5ee03c08ed3936348eeb3931be8f804e61f2b09debf305967c6a7bbf007e0", "firstIndex": 1543502599},
+{"blockNumber": 5155029, "blockId": "0x84f590dfc2e11f1ca53c1757ac3c508d56f55ee24d6ca5501895974be4250d76", "firstIndex": 1610605837},
+{"blockNumber": 5204413, "blockId": "0xeaf2c3fb6f927c16d38fab08b34303867b87470558612404c7f9e3256b80c5b9", "firstIndex": 1677720841},
+{"blockNumber": 5269957, "blockId": "0x596e0b2e8e4c18c803b61767320fe32c063153d870c94e4a08e9a68cbaa582a9", "firstIndex": 1744825147},
+{"blockNumber": 5337678, "blockId": "0x7b2d54f8af1ecaaaab994e137d4421d8236c1c10d9a7bdcb9e5500db7a3fe9a3", "firstIndex": 1811939316},
+{"blockNumber": 5399058, "blockId": "0xb61ef16d55c96682fb62b0110a2dbc50d8eff2526be4121ece3690700611c71b", "firstIndex": 1879046044},
+{"blockNumber": 5422707, "blockId": "0xdabcab7c0cc9cb9f22f7507a1076c87831cb1afed9d0aa5bcd93f22266720c91", "firstIndex": 1946156915},
+{"blockNumber": 5454264, "blockId": "0xe1bde812906605ce662f5fd9f01b49c7331fb25f52ab5b12d35ea2b4da5458fe", "firstIndex": 2013259168},
+{"blockNumber": 5498898, "blockId": "0x9533d9c5353d22f8a235e95831cfbf4d5a7220a430ca23494526a9d3aa866fe8", "firstIndex": 2080374321},
+{"blockNumber": 5554801, "blockId": "0xe7b320bbecb19f1e99dd6ce4aed1efc754d7b2022e1f80389e8a21413c465f55", "firstIndex": 2147476253},
+{"blockNumber": 5594725, "blockId": "0xce6750be4a5b3e0fe152dd02308e94f7d56b254852a7e9acef6e14105053d7d1", "firstIndex": 2214591591},
+{"blockNumber": 5645198, "blockId": "0x5d42d39999c546f37001d5f613732fb54032384dd71a686d3664d2c8a1337752", "firstIndex": 2281696503},
+{"blockNumber": 5687659, "blockId": "0x3ed941be39a33ffa69cf3531a67f5a25f712ba05db890ff377f60d26842e4b1c", "firstIndex": 2348801751},
+{"blockNumber": 5727823, "blockId": "0xaf699b6c4cd58181bd609a66990b8edb5d1b94d5ff1ab732ded35ce7b8373353", "firstIndex": 2415917178},
+{"blockNumber": 5784505, "blockId": "0x621c740d04ea41f70a2f0537e21e5b96169aea8a8efee0ae5527717e5c40aa64", "firstIndex": 2483024581},
+{"blockNumber": 5843958, "blockId": "0xec122204a4e4698748f55a1c9f8582c46bacda029aee4de1a234e67e3288e6b1", "firstIndex": 2550136761},
+{"blockNumber": 5906359, "blockId": "0x8af5ce73fbd7a6110fb8b19b75a7322456ece88fcfa1614c745f1a65f4e915c1", "firstIndex": 2617245617},
+{"blockNumber": 5977944, "blockId": "0xbc8186258298a4f376124989cfb7b22c2bea6603a5245bb6c505c5fc45844bbd", "firstIndex": 2684350982},
+{"blockNumber": 6051571, "blockId": "0x54f9df9d9d73d1aa1cfcd6f372377c6013ecba2a1ed158d3c304f4fca51dae58", "firstIndex": 2751463209},
+{"blockNumber": 6118461, "blockId": "0xfea757fad3f763c62a514a9c24924934539ca56620bd811f83e9cc2e671f0cf0", "firstIndex": 2818572283},
+{"blockNumber": 6174385, "blockId": "0x2d8d0226e58f7516c13f9e1c9cf3ea65bb520fa1dfd7249dc9ea34a4e1fd430d", "firstIndex": 2885681036},
+{"blockNumber": 6276318, "blockId": "0xa922e9d54fd062b658c4866ed218632ddd51f250d671628a42968bb912d3ed5d", "firstIndex": 2952789983},
+{"blockNumber": 6368452, "blockId": "0x8d3d7466a7c9ca7298f82c37c38b0f64ec04522d2ed2e2349f8edc020c57f2c4", "firstIndex": 3019898695},
+{"blockNumber": 6470810, "blockId": "0x9887c35542835ee81153fa0e4d8a9e6f170b6e14fc78d8c7f3d900d0a70434f1", "firstIndex": 3087007578},
+{"blockNumber": 6553334, "blockId": "0x7b0d89a0282c18785fcc108dbdc9d45dd9d63b7084ddc676df9e9504585a5969", "firstIndex": 3154115987},
+{"blockNumber": 6663825, "blockId": "0xff6cec99324a89d6d36275c17a4569f0cba203fe5b0350f155a7d5445e0ed419", "firstIndex": 3221224775},
+{"blockNumber": 6767082, "blockId": "0xe10a96a7194f98bf262f0cb1cdfb4d3b9a2072139dfcbe3f1eb01419e353044e", "firstIndex": 3288334139},
+{"blockNumber": 6886709, "blockId": "0x20f6a5d986913025ad5b6b6387d818e49a3caf838326f4002c1439ca61313be5", "firstIndex": 3355442979},
+{"blockNumber": 6978948, "blockId": "0xd7c3024765245ec49e6a48b076d540bc91f57f2ccc125e17d60dd37bb669f843", "firstIndex": 3422551908},
+{"blockNumber": 7098891, "blockId": "0x05114c037e1b4d69a46d74a974be9bce45e87ad2226a59b44dd17f98dd2fd0d1", "firstIndex": 3489659530},
+{"blockNumber": 7203157, "blockId": "0xc0f610014fcd9f2850274b58179d474f0947676fd0639b2884316467c631811d", "firstIndex": 3556769512},
+{"blockNumber": 7256735, "blockId": "0x0324c15b3b23fd82c2962dd167618e77e60ebeac5a2c87f672caddc9732337b3", "firstIndex": 3623876508},
+{"blockNumber": 7307851, "blockId": "0x8e23280d1a3aec877d7758413ed20299d381aa43e7e2fc6f381ad96e8ff0acef", "firstIndex": 3690987098},
+{"blockNumber": 7369389, "blockId": "0xbf6436eb2b88539945d6673141a14cb79ffc1e7db2b57176acf8e02ff3b6fcd3", "firstIndex": 3758096287},
+{"blockNumber": 7445220, "blockId": "0x147619f74815283d834ac08ff494fb4791207b3949c64b2623f11ff6141ee7a7", "firstIndex": 3825204992},
+{"blockNumber": 7511632, "blockId": "0x5094d64868f419e6ac3d253d19d5feda76564a0d56d7bbf8a822dff1c2261b30", "firstIndex": 3892314047},
+{"blockNumber": 7557280, "blockId": "0x54aba9351a1ba51873645221aa7c991024da1fe468a600ddb6e2559351d9c28f", "firstIndex": 3959422859},
+{"blockNumber": 7606304, "blockId": "0xbbe2fed08cf0b0ff2cb6ae9fd7257843f77a04a7d4cafb06d7a4bedea6ab0c98", "firstIndex": 4026531690}
+]
+
diff --git a/core/filtermaps/filtermaps.go b/core/filtermaps/filtermaps.go
new file mode 100644
index 00000000000..95dd1815519
--- /dev/null
+++ b/core/filtermaps/filtermaps.go
@@ -0,0 +1,726 @@
+// Copyright 2024 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package filtermaps
+
+import (
+ "bytes"
+ "errors"
+ "fmt"
+ "os"
+ "slices"
+ "sync"
+ "time"
+
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/common/lru"
+ "github.com/ethereum/go-ethereum/core/rawdb"
+ "github.com/ethereum/go-ethereum/core/types"
+ "github.com/ethereum/go-ethereum/ethdb"
+ "github.com/ethereum/go-ethereum/ethdb/leveldb"
+ "github.com/ethereum/go-ethereum/log"
+)
+
+const (
+ cachedLastBlocks = 1000 // last block of map pointers
+ cachedLvPointers = 1000 // first log value pointer of block pointers
+ cachedBaseRows = 100 // groups of base layer filter row data
+ cachedFilterMaps = 3 // complete filter maps (cached by map renderer)
+ cachedRenderSnapshots = 8 // saved map renderer data at block boundaries
+)
+
+// FilterMaps is the in-memory representation of the log index structure that is
+// responsible for building and updating the index according to the canonical
+// chain.
+//
+// Note that FilterMaps implements the same data structure as proposed in EIP-7745
+// without the tree hashing and consensus changes:
+// https://eips.ethereum.org/EIPS/eip-7745
+type FilterMaps struct {
+ // If disabled is set, log indexing is fully disabled.
+ // This is configured by the --history.logs.disable Geth flag.
+ // We chose to implement disabling this way because it requires less special
+ // case logic in eth/filters.
+ disabled bool
+
+ closeCh chan struct{}
+ closeWg sync.WaitGroup
+ history uint64
+ exportFileName string
+ Params
+
+ db ethdb.KeyValueStore
+
+ // fields written by the indexer and read by matcher backend. Indexer can
+ // read them without a lock and write them under indexLock write lock.
+ // Matcher backend can read them under indexLock read lock.
+ indexLock sync.RWMutex
+ indexedRange filterMapsRange
+ indexedView *ChainView // always consistent with the log index
+
+ // also accessed by indexer and matcher backend but no locking needed.
+ filterMapCache *lru.Cache[uint32, filterMap]
+ lastBlockCache *lru.Cache[uint32, lastBlockOfMap]
+ lvPointerCache *lru.Cache[uint64, uint64]
+ baseRowsCache *lru.Cache[uint64, [][]uint32]
+
+ // the matchers set and the fields of FilterMapsMatcherBackend instances are
+ // read and written both by exported functions and the indexer.
+ // Note that if both indexLock and matchersLock needs to be locked then
+ // indexLock should be locked first.
+ matchersLock sync.Mutex
+ matchers map[*FilterMapsMatcherBackend]struct{}
+
+ // fields only accessed by the indexer (no mutex required).
+ renderSnapshots *lru.Cache[uint64, *renderedMap]
+ startedHeadIndex, startedTailIndex, startedTailUnindex bool
+ startedHeadIndexAt, startedTailIndexAt, startedTailUnindexAt time.Time
+ loggedHeadIndex, loggedTailIndex bool
+ lastLogHeadIndex, lastLogTailIndex time.Time
+ ptrHeadIndex, ptrTailIndex, ptrTailUnindexBlock uint64
+ ptrTailUnindexMap uint32
+
+ targetView *ChainView
+ matcherSyncRequest *FilterMapsMatcherBackend
+ finalBlock, lastFinal uint64
+ lastFinalEpoch uint32
+ stop bool
+ targetViewCh chan *ChainView
+ finalBlockCh chan uint64
+ blockProcessingCh chan bool
+ blockProcessing bool
+ matcherSyncCh chan *FilterMapsMatcherBackend
+ waitIdleCh chan chan bool
+ tailRenderer *mapRenderer
+
+ // test hooks
+ testDisableSnapshots, testSnapshotUsed bool
+}
+
+// filterMap is a full or partial in-memory representation of a filter map where
+// rows are allowed to have a nil value meaning the row is not stored in the
+// structure. Note that therefore a known empty row should be represented with
+// a zero-length slice.
+// It can be used as a memory cache or an overlay while preparing a batch of
+// changes to the structure. In either case a nil value should be interpreted
+// as transparent (uncached/unchanged).
+type filterMap []FilterRow
+
+// copy returns a copy of the given filter map. Note that the row slices are
+// copied but their contents are not. This permits extending the rows further
+// (which happens during map rendering) without affecting the validity of
+// copies made for snapshots during rendering.
+func (fm filterMap) copy() filterMap {
+ c := make(filterMap, len(fm))
+ copy(c, fm)
+ return c
+}
+
+// FilterRow encodes a single row of a filter map as a list of column indices.
+// Note that the values are always stored in the same order as they were added
+// and if the same column index is added twice, it is also stored twice.
+// Order of column indices and potential duplications do not matter when searching
+// for a value but leaving the original order makes reverting to a previous state
+// simpler.
+type FilterRow []uint32
+
+// Equal returns true if the given filter rows are equivalent.
+func (a FilterRow) Equal(b FilterRow) bool {
+ return slices.Equal(a, b)
+}
+
+// filterMapsRange describes the rendered range of filter maps and the range
+// of fully rendered blocks.
+type filterMapsRange struct {
+ initialized bool
+ headBlockIndexed bool
+ headBlockDelimiter uint64 // zero if afterLastIndexedBlock != targetBlockNumber
+ // if initialized then all maps are rendered between firstRenderedMap and
+ // afterLastRenderedMap-1
+ firstRenderedMap, afterLastRenderedMap uint32
+ // if tailPartialEpoch > 0 then maps between firstRenderedMap-mapsPerEpoch and
+ // firstRenderedMap-mapsPerEpoch+tailPartialEpoch-1 are rendered
+ tailPartialEpoch uint32
+ // if initialized then all log values belonging to blocks between
+ // firstIndexedBlock and afterLastIndexedBlock are fully rendered
+ // blockLvPointers are available between firstIndexedBlock and afterLastIndexedBlock-1
+ firstIndexedBlock, afterLastIndexedBlock uint64
+}
+
+// hasIndexedBlocks returns true if the range has at least one fully indexed block.
+func (fmr *filterMapsRange) hasIndexedBlocks() bool {
+ return fmr.initialized && fmr.afterLastIndexedBlock > fmr.firstIndexedBlock
+}
+
+// lastBlockOfMap is used for caching the (number, id) pairs belonging to the
+// last block of each map.
+type lastBlockOfMap struct {
+ number uint64
+ id common.Hash
+}
+
+// Config contains the configuration options for NewFilterMaps.
+type Config struct {
+ History uint64 // number of historical blocks to index
+ Disabled bool // disables indexing completely
+
+ // This option enables the checkpoint JSON file generator.
+ // If set, the given file will be updated with checkpoint information.
+ ExportFileName string
+}
+
+// NewFilterMaps creates a new FilterMaps and starts the indexer.
+func NewFilterMaps(db ethdb.KeyValueStore, initView *ChainView, params Params, config Config) *FilterMaps {
+ rs, initialized, err := rawdb.ReadFilterMapsRange(db)
+ if err != nil {
+ log.Error("Error reading log index range", "error", err)
+ }
+ params.deriveFields()
+ f := &FilterMaps{
+ db: db,
+ closeCh: make(chan struct{}),
+ waitIdleCh: make(chan chan bool),
+ targetViewCh: make(chan *ChainView, 1),
+ finalBlockCh: make(chan uint64, 1),
+ blockProcessingCh: make(chan bool, 1),
+ history: config.History,
+ disabled: config.Disabled,
+ exportFileName: config.ExportFileName,
+ Params: params,
+ indexedRange: filterMapsRange{
+ initialized: initialized,
+ headBlockIndexed: rs.HeadBlockIndexed,
+ headBlockDelimiter: rs.HeadBlockDelimiter,
+ firstIndexedBlock: rs.FirstIndexedBlock,
+ afterLastIndexedBlock: rs.AfterLastIndexedBlock,
+ firstRenderedMap: rs.FirstRenderedMap,
+ afterLastRenderedMap: rs.AfterLastRenderedMap,
+ tailPartialEpoch: rs.TailPartialEpoch,
+ },
+ matcherSyncCh: make(chan *FilterMapsMatcherBackend),
+ matchers: make(map[*FilterMapsMatcherBackend]struct{}),
+ filterMapCache: lru.NewCache[uint32, filterMap](cachedFilterMaps),
+ lastBlockCache: lru.NewCache[uint32, lastBlockOfMap](cachedLastBlocks),
+ lvPointerCache: lru.NewCache[uint64, uint64](cachedLvPointers),
+ baseRowsCache: lru.NewCache[uint64, [][]uint32](cachedBaseRows),
+ renderSnapshots: lru.NewCache[uint64, *renderedMap](cachedRenderSnapshots),
+ }
+
+ // Set initial indexer target.
+ f.targetView = initView
+ if f.indexedRange.initialized {
+ f.indexedView = f.initChainView(f.targetView)
+ f.indexedRange.headBlockIndexed = f.indexedRange.afterLastIndexedBlock == f.indexedView.headNumber+1
+ if !f.indexedRange.headBlockIndexed {
+ f.indexedRange.headBlockDelimiter = 0
+ }
+ }
+ if f.indexedRange.hasIndexedBlocks() {
+ log.Info("Initialized log indexer",
+ "first block", f.indexedRange.firstIndexedBlock, "last block", f.indexedRange.afterLastIndexedBlock-1,
+ "first map", f.indexedRange.firstRenderedMap, "last map", f.indexedRange.afterLastRenderedMap-1,
+ "head indexed", f.indexedRange.headBlockIndexed)
+ }
+ return f
+}
+
+// Start starts the indexer.
+func (f *FilterMaps) Start() {
+ if !f.testDisableSnapshots && f.indexedRange.initialized && f.indexedRange.headBlockIndexed &&
+ f.indexedRange.firstRenderedMap < f.indexedRange.afterLastRenderedMap {
+ // previous target head rendered; load last map as snapshot
+ if err := f.loadHeadSnapshot(); err != nil {
+ log.Error("Could not load head filter map snapshot", "error", err)
+ }
+ }
+ f.closeWg.Add(1)
+ go f.indexerLoop()
+}
+
+// Stop ensures that the indexer is fully stopped before returning.
+func (f *FilterMaps) Stop() {
+ close(f.closeCh)
+ f.closeWg.Wait()
+}
+
+// initChainView returns a chain view consistent with both the current target
+// view and the current state of the log index as found in the database, based
+// on the last block of stored maps.
+// Note that the returned view might be shorter than the existing index if
+// the latest maps are not consistent with targetView.
+func (f *FilterMaps) initChainView(chainView *ChainView) *ChainView {
+ mapIndex := f.indexedRange.afterLastRenderedMap
+ for {
+ var ok bool
+ mapIndex, ok = f.lastMapBoundaryBefore(mapIndex)
+ if !ok {
+ break
+ }
+ lastBlockNumber, lastBlockId, err := f.getLastBlockOfMap(mapIndex)
+ if err != nil {
+ log.Error("Could not initialize indexed chain view", "error", err)
+ break
+ }
+ if lastBlockNumber <= chainView.headNumber && chainView.getBlockId(lastBlockNumber) == lastBlockId {
+ return chainView.limitedView(lastBlockNumber)
+ }
+ }
+ return chainView.limitedView(0)
+}
+
+// reset un-initializes the FilterMaps structure and removes all related data from
+// the database. The function returns true if everything was successfully removed.
+func (f *FilterMaps) reset() bool {
+ f.indexLock.Lock()
+ f.indexedRange = filterMapsRange{}
+ f.indexedView = nil
+ f.filterMapCache.Purge()
+ f.renderSnapshots.Purge()
+ f.lastBlockCache.Purge()
+ f.lvPointerCache.Purge()
+ f.baseRowsCache.Purge()
+ f.indexLock.Unlock()
+ // deleting the range first ensures that resetDb will be called again at next
+ // startup and any leftover data will be removed even if it cannot finish now.
+ rawdb.DeleteFilterMapsRange(f.db)
+ return f.removeDbWithPrefix([]byte(rawdb.FilterMapsPrefix), "Resetting log index database")
+}
+
+// init initializes an empty log index according to the current targetView.
+func (f *FilterMaps) init() error {
+ f.indexLock.Lock()
+ defer f.indexLock.Unlock()
+
+ var bestIdx, bestLen int
+ for idx, checkpointList := range checkpoints {
+ // binary search for the last matching epoch head
+ min, max := 0, len(checkpointList)
+ for min < max {
+ mid := (min + max + 1) / 2
+ cp := checkpointList[mid-1]
+ if cp.BlockNumber <= f.targetView.headNumber && f.targetView.getBlockId(cp.BlockNumber) == cp.BlockId {
+ min = mid
+ } else {
+ max = mid - 1
+ }
+ }
+ if max > bestLen {
+ bestIdx, bestLen = idx, max
+ }
+ }
+ batch := f.db.NewBatch()
+ for epoch := range bestLen {
+ cp := checkpoints[bestIdx][epoch]
+ f.storeLastBlockOfMap(batch, (uint32(epoch+1)< 0 {
+ cp := checkpoints[bestIdx][bestLen-1]
+ fmr.firstIndexedBlock = cp.BlockNumber + 1
+ fmr.afterLastIndexedBlock = cp.BlockNumber + 1
+ fmr.firstRenderedMap = uint32(bestLen) << f.logMapsPerEpoch
+ fmr.afterLastRenderedMap = uint32(bestLen) << f.logMapsPerEpoch
+ }
+ f.setRange(batch, f.targetView, fmr)
+ return batch.Write()
+}
+
+// removeDbWithPrefix removes data with the given prefix from the database and
+// returns true if everything was successfully removed.
+func (f *FilterMaps) removeDbWithPrefix(prefix []byte, action string) bool {
+ it := f.db.NewIterator(prefix, nil)
+ hasData := it.Next()
+ it.Release()
+ if !hasData {
+ return true
+ }
+
+ end := bytes.Clone(prefix)
+ end[len(end)-1]++
+ start := time.Now()
+ var retry bool
+ for {
+ err := f.db.DeleteRange(prefix, end)
+ if err == nil {
+ log.Info(action+" finished", "elapsed", time.Since(start))
+ return true
+ }
+ if err != leveldb.ErrTooManyKeys {
+ log.Error(action+" failed", "error", err)
+ return false
+ }
+ select {
+ case <-f.closeCh:
+ return false
+ default:
+ }
+ if !retry {
+ log.Info(action + " in progress...")
+ retry = true
+ }
+ }
+}
+
+// setRange updates the indexed chain view and covered range and also adds the
+// changes to the given batch.
+// Note that this function assumes that the index write lock is being held.
+func (f *FilterMaps) setRange(batch ethdb.KeyValueWriter, newView *ChainView, newRange filterMapsRange) {
+ f.indexedView = newView
+ f.indexedRange = newRange
+ f.updateMatchersValidRange()
+ if newRange.initialized {
+ rs := rawdb.FilterMapsRange{
+ HeadBlockIndexed: newRange.headBlockIndexed,
+ HeadBlockDelimiter: newRange.headBlockDelimiter,
+ FirstIndexedBlock: newRange.firstIndexedBlock,
+ AfterLastIndexedBlock: newRange.afterLastIndexedBlock,
+ FirstRenderedMap: newRange.firstRenderedMap,
+ AfterLastRenderedMap: newRange.afterLastRenderedMap,
+ TailPartialEpoch: newRange.tailPartialEpoch,
+ }
+ rawdb.WriteFilterMapsRange(batch, rs)
+ } else {
+ rawdb.DeleteFilterMapsRange(batch)
+ }
+}
+
+// getLogByLvIndex returns the log at the given log value index. If the index does
+// not point to the first log value entry of a log then no log and no error are
+// returned as this can happen when the log value index was a false positive.
+// Note that this function assumes that the log index structure is consistent
+// with the canonical chain at the point where the given log value index points.
+// If this is not the case then an invalid result or an error may be returned.
+// Note that this function assumes that the indexer read lock is being held when
+// called from outside the indexerLoop goroutine.
+func (f *FilterMaps) getLogByLvIndex(lvIndex uint64) (*types.Log, error) {
+ mapIndex := uint32(lvIndex >> f.logValuesPerMap)
+ if mapIndex < f.indexedRange.firstRenderedMap || mapIndex >= f.indexedRange.afterLastRenderedMap {
+ return nil, nil
+ }
+ // find possible block range based on map to block pointers
+ lastBlockNumber, _, err := f.getLastBlockOfMap(mapIndex)
+ if err != nil {
+ return nil, fmt.Errorf("failed to retrieve last block of map %d containing searched log value index %d: %v", mapIndex, lvIndex, err)
+ }
+ var firstBlockNumber uint64
+ if mapIndex > 0 {
+ firstBlockNumber, _, err = f.getLastBlockOfMap(mapIndex - 1)
+ if err != nil {
+ return nil, fmt.Errorf("failed to retrieve last block of map %d before searched log value index %d: %v", mapIndex, lvIndex, err)
+ }
+ }
+ if firstBlockNumber < f.indexedRange.firstIndexedBlock {
+ firstBlockNumber = f.indexedRange.firstIndexedBlock
+ }
+ // find block with binary search based on block to log value index pointers
+ for firstBlockNumber < lastBlockNumber {
+ midBlockNumber := (firstBlockNumber + lastBlockNumber + 1) / 2
+ midLvPointer, err := f.getBlockLvPointer(midBlockNumber)
+ if err != nil {
+ return nil, fmt.Errorf("failed to retrieve log value pointer of block %d while binary searching log value index %d: %v", midBlockNumber, lvIndex, err)
+ }
+ if lvIndex < midLvPointer {
+ lastBlockNumber = midBlockNumber - 1
+ } else {
+ firstBlockNumber = midBlockNumber
+ }
+ }
+ // get block receipts
+ receipts := f.indexedView.getReceipts(firstBlockNumber)
+ if receipts == nil {
+ return nil, fmt.Errorf("failed to retrieve receipts for block %d containing searched log value index %d: %v", firstBlockNumber, lvIndex, err)
+ }
+ lvPointer, err := f.getBlockLvPointer(firstBlockNumber)
+ if err != nil {
+ return nil, fmt.Errorf("failed to retrieve log value pointer of block %d containing searched log value index %d: %v", firstBlockNumber, lvIndex, err)
+ }
+ // iterate through receipts to find the exact log starting at lvIndex
+ for _, receipt := range receipts {
+ for _, log := range receipt.Logs {
+ if lvPointer > lvIndex {
+ // lvIndex does not point to the first log value (address value)
+ // generated by a log as true matches should always do, so it
+ // is considered a false positive (no log and no error returned).
+ return nil, nil
+ }
+ if lvPointer == lvIndex {
+ return log, nil // potential match
+ }
+ lvPointer += uint64(len(log.Topics) + 1)
+ }
+ }
+ return nil, nil
+}
+
+// getFilterMap fetches an entire filter map from the database.
+func (f *FilterMaps) getFilterMap(mapIndex uint32) (filterMap, error) {
+ if fm, ok := f.filterMapCache.Get(mapIndex); ok {
+ return fm, nil
+ }
+ fm := make(filterMap, f.mapHeight)
+ for rowIndex := range fm {
+ var err error
+ fm[rowIndex], err = f.getFilterMapRow(mapIndex, uint32(rowIndex), false)
+ if err != nil {
+ return nil, fmt.Errorf("failed to load filter map %d from database: %v", mapIndex, err)
+ }
+ }
+ f.filterMapCache.Add(mapIndex, fm)
+ return fm, nil
+}
+
+// getFilterMapRow fetches the given filter map row. If baseLayerOnly is true
+// then only the first baseRowLength entries are returned.
+func (f *FilterMaps) getFilterMapRow(mapIndex, rowIndex uint32, baseLayerOnly bool) (FilterRow, error) {
+ baseMapRowIndex := f.mapRowIndex(mapIndex&-f.baseRowGroupLength, rowIndex)
+ baseRows, ok := f.baseRowsCache.Get(baseMapRowIndex)
+ if !ok {
+ var err error
+ baseRows, err = rawdb.ReadFilterMapBaseRows(f.db, baseMapRowIndex, f.baseRowGroupLength, f.logMapWidth)
+ if err != nil {
+ return nil, fmt.Errorf("failed to retrieve filter map %d base rows %d: %v", mapIndex, rowIndex, err)
+ }
+ f.baseRowsCache.Add(baseMapRowIndex, baseRows)
+ }
+ baseRow := baseRows[mapIndex&(f.baseRowGroupLength-1)]
+ if baseLayerOnly {
+ return baseRow, nil
+ }
+ extRow, err := rawdb.ReadFilterMapExtRow(f.db, f.mapRowIndex(mapIndex, rowIndex), f.logMapWidth)
+ if err != nil {
+ return nil, fmt.Errorf("failed to retrieve filter map %d extended row %d: %v", mapIndex, rowIndex, err)
+ }
+ return FilterRow(append(baseRow, extRow...)), nil
+}
+
+// storeFilterMapRows stores a set of filter map rows at the corresponding map
+// indices and a shared row index.
+func (f *FilterMaps) storeFilterMapRows(batch ethdb.Batch, mapIndices []uint32, rowIndex uint32, rows []FilterRow) error {
+ for len(mapIndices) > 0 {
+ baseMapIndex := mapIndices[0] & -f.baseRowGroupLength
+ groupLength := 1
+ for groupLength < len(mapIndices) && mapIndices[groupLength]&-f.baseRowGroupLength == baseMapIndex {
+ groupLength++
+ }
+ if err := f.storeFilterMapRowsOfGroup(batch, mapIndices[:groupLength], rowIndex, rows[:groupLength]); err != nil {
+ return err
+ }
+ mapIndices, rows = mapIndices[groupLength:], rows[groupLength:]
+ }
+ return nil
+}
+
+// storeFilterMapRowsOfGroup stores a set of filter map rows at map indices
+// belonging to the same base row group.
+func (f *FilterMaps) storeFilterMapRowsOfGroup(batch ethdb.Batch, mapIndices []uint32, rowIndex uint32, rows []FilterRow) error {
+ baseMapIndex := mapIndices[0] & -f.baseRowGroupLength
+ baseMapRowIndex := f.mapRowIndex(baseMapIndex, rowIndex)
+ var baseRows [][]uint32
+ if uint32(len(mapIndices)) != f.baseRowGroupLength { // skip base rows read if all rows are replaced
+ var ok bool
+ baseRows, ok = f.baseRowsCache.Get(baseMapRowIndex)
+ if !ok {
+ var err error
+ baseRows, err = rawdb.ReadFilterMapBaseRows(f.db, baseMapRowIndex, f.baseRowGroupLength, f.logMapWidth)
+ if err != nil {
+ return fmt.Errorf("failed to retrieve filter map %d base rows %d for modification: %v", mapIndices[0]&-f.baseRowGroupLength, rowIndex, err)
+ }
+ }
+ } else {
+ baseRows = make([][]uint32, f.baseRowGroupLength)
+ }
+ for i, mapIndex := range mapIndices {
+ if mapIndex&-f.baseRowGroupLength != baseMapIndex {
+ panic("mapIndices are not in the same base row group")
+ }
+ baseRow := []uint32(rows[i])
+ var extRow FilterRow
+ if uint32(len(rows[i])) > f.baseRowLength {
+ extRow = baseRow[f.baseRowLength:]
+ baseRow = baseRow[:f.baseRowLength]
+ }
+ baseRows[mapIndex&(f.baseRowGroupLength-1)] = baseRow
+ rawdb.WriteFilterMapExtRow(batch, f.mapRowIndex(mapIndex, rowIndex), extRow, f.logMapWidth)
+ }
+ f.baseRowsCache.Add(baseMapRowIndex, baseRows)
+ rawdb.WriteFilterMapBaseRows(batch, baseMapRowIndex, baseRows, f.logMapWidth)
+ return nil
+}
+
+// mapRowIndex calculates the unified storage index where the given row of the
+// given map is stored. Note that this indexing scheme is the same as the one
+// proposed in EIP-7745 for tree-hashing the filter map structure and for the
+// same data proximity reasons it is also suitable for database representation.
+// See also:
+// https://eips.ethereum.org/EIPS/eip-7745#hash-tree-structure
+func (f *FilterMaps) mapRowIndex(mapIndex, rowIndex uint32) uint64 {
+ epochIndex, mapSubIndex := mapIndex>>f.logMapsPerEpoch, mapIndex&(f.mapsPerEpoch-1)
+ return (uint64(epochIndex)<= f.indexedRange.afterLastIndexedBlock && f.indexedRange.headBlockIndexed {
+ return f.indexedRange.headBlockDelimiter, nil
+ }
+ if lvPointer, ok := f.lvPointerCache.Get(blockNumber); ok {
+ return lvPointer, nil
+ }
+ lvPointer, err := rawdb.ReadBlockLvPointer(f.db, blockNumber)
+ if err != nil {
+ return 0, fmt.Errorf("failed to retrieve log value pointer of block %d: %v", blockNumber, err)
+ }
+ f.lvPointerCache.Add(blockNumber, lvPointer)
+ return lvPointer, nil
+}
+
+// storeBlockLvPointer stores the starting log value index where the log values
+// generated by the given block are located.
+func (f *FilterMaps) storeBlockLvPointer(batch ethdb.Batch, blockNumber, lvPointer uint64) {
+ f.lvPointerCache.Add(blockNumber, lvPointer)
+ rawdb.WriteBlockLvPointer(batch, blockNumber, lvPointer)
+}
+
+// deleteBlockLvPointer deletes the starting log value index where the log values
+// generated by the given block are located.
+func (f *FilterMaps) deleteBlockLvPointer(batch ethdb.Batch, blockNumber uint64) {
+ f.lvPointerCache.Remove(blockNumber)
+ rawdb.DeleteBlockLvPointer(batch, blockNumber)
+}
+
+// getLastBlockOfMap returns the number and id of the block that generated the
+// last log value entry of the given map.
+func (f *FilterMaps) getLastBlockOfMap(mapIndex uint32) (uint64, common.Hash, error) {
+ if lastBlock, ok := f.lastBlockCache.Get(mapIndex); ok {
+ return lastBlock.number, lastBlock.id, nil
+ }
+ number, id, err := rawdb.ReadFilterMapLastBlock(f.db, mapIndex)
+ if err != nil {
+ return 0, common.Hash{}, fmt.Errorf("failed to retrieve last block of map %d: %v", mapIndex, err)
+ }
+ f.lastBlockCache.Add(mapIndex, lastBlockOfMap{number: number, id: id})
+ return number, id, nil
+}
+
+// storeLastBlockOfMap stores the number of the block that generated the last
+// log value entry of the given map.
+func (f *FilterMaps) storeLastBlockOfMap(batch ethdb.Batch, mapIndex uint32, number uint64, id common.Hash) {
+ f.lastBlockCache.Add(mapIndex, lastBlockOfMap{number: number, id: id})
+ rawdb.WriteFilterMapLastBlock(batch, mapIndex, number, id)
+}
+
+// deleteLastBlockOfMap deletes the number of the block that generated the last
+// log value entry of the given map.
+func (f *FilterMaps) deleteLastBlockOfMap(batch ethdb.Batch, mapIndex uint32) {
+ f.lastBlockCache.Remove(mapIndex)
+ rawdb.DeleteFilterMapLastBlock(batch, mapIndex)
+}
+
+// deleteTailEpoch deletes index data from the earliest, either fully or partially
+// indexed epoch. The last block pointer for the last map of the epoch and the
+// corresponding block log value pointer are retained as these are always assumed
+// to be available for each epoch.
+func (f *FilterMaps) deleteTailEpoch(epoch uint32) error {
+ f.indexLock.Lock()
+ defer f.indexLock.Unlock()
+
+ firstMap := epoch << f.logMapsPerEpoch
+ lastBlock, _, err := f.getLastBlockOfMap(firstMap + f.mapsPerEpoch - 1)
+ if err != nil {
+ return fmt.Errorf("failed to retrieve last block of deleted epoch %d: %v", epoch, err)
+ }
+ var firstBlock uint64
+ if epoch > 0 {
+ firstBlock, _, err = f.getLastBlockOfMap(firstMap - 1)
+ if err != nil {
+ return fmt.Errorf("failed to retrieve last block before deleted epoch %d: %v", epoch, err)
+ }
+ firstBlock++
+ }
+ fmr := f.indexedRange
+ if f.indexedRange.firstRenderedMap == firstMap &&
+ f.indexedRange.afterLastRenderedMap > firstMap+f.mapsPerEpoch &&
+ f.indexedRange.tailPartialEpoch == 0 {
+ fmr.firstRenderedMap = firstMap + f.mapsPerEpoch
+ fmr.firstIndexedBlock = lastBlock + 1
+ } else if f.indexedRange.firstRenderedMap == firstMap+f.mapsPerEpoch {
+ fmr.tailPartialEpoch = 0
+ } else {
+ return errors.New("invalid tail epoch number")
+ }
+ f.setRange(f.db, f.indexedView, fmr)
+ rawdb.DeleteFilterMapRows(f.db, f.mapRowIndex(firstMap, 0), f.mapRowIndex(firstMap+f.mapsPerEpoch, 0))
+ for mapIndex := firstMap; mapIndex < firstMap+f.mapsPerEpoch; mapIndex++ {
+ f.filterMapCache.Remove(mapIndex)
+ }
+ rawdb.DeleteFilterMapLastBlocks(f.db, firstMap, firstMap+f.mapsPerEpoch-1) // keep last enrty
+ for mapIndex := firstMap; mapIndex < firstMap+f.mapsPerEpoch-1; mapIndex++ {
+ f.lastBlockCache.Remove(mapIndex)
+ }
+ rawdb.DeleteBlockLvPointers(f.db, firstBlock, lastBlock) // keep last enrty
+ for blockNumber := firstBlock; blockNumber < lastBlock; blockNumber++ {
+ f.lvPointerCache.Remove(blockNumber)
+ }
+ return nil
+}
+
+// exportCheckpoints exports epoch checkpoints in the format used by checkpoints.go.
+func (f *FilterMaps) exportCheckpoints() {
+ finalLvPtr, err := f.getBlockLvPointer(f.finalBlock + 1)
+ if err != nil {
+ log.Error("Error fetching log value pointer of finalized block", "block", f.finalBlock, "error", err)
+ return
+ }
+ epochCount := uint32(finalLvPtr >> (f.logValuesPerMap + f.logMapsPerEpoch))
+ if epochCount == f.lastFinalEpoch {
+ return
+ }
+ w, err := os.Create(f.exportFileName)
+ if err != nil {
+ log.Error("Error creating checkpoint export file", "name", f.exportFileName, "error", err)
+ return
+ }
+ defer w.Close()
+
+ log.Info("Exporting log index checkpoints", "epochs", epochCount, "file", f.exportFileName)
+ w.WriteString("[\n")
+ comma := ","
+ for epoch := uint32(0); epoch < epochCount; epoch++ {
+ lastBlock, lastBlockId, err := f.getLastBlockOfMap((epoch+1)<.
+
+package filtermaps
+
+import (
+ "math"
+ "time"
+
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/log"
+)
+
+const (
+ logFrequency = time.Second * 20 // log info frequency during long indexing/unindexing process
+ headLogDelay = time.Second // head indexing log info delay (do not log if finished faster)
+)
+
+// updateLoop initializes and updates the log index structure according to the
+// current targetView.
+func (f *FilterMaps) indexerLoop() {
+ defer f.closeWg.Done()
+
+ if f.disabled {
+ f.reset()
+ return
+ }
+ log.Info("Started log indexer")
+
+ for !f.stop {
+ if !f.indexedRange.initialized {
+ if err := f.init(); err != nil {
+ log.Error("Error initializing log index", "error", err)
+ f.waitForEvent()
+ continue
+ }
+ }
+ if !f.targetHeadIndexed() {
+ if !f.tryIndexHead() {
+ f.waitForEvent()
+ }
+ } else {
+ if f.finalBlock != f.lastFinal {
+ if f.exportFileName != "" {
+ f.exportCheckpoints()
+ }
+ f.lastFinal = f.finalBlock
+ }
+ if f.tryIndexTail() && f.tryUnindexTail() {
+ f.waitForEvent()
+ }
+ }
+ }
+}
+
+// SetTargetView sets a new target chain view for the indexer to render.
+// Note that SetTargetView never blocks.
+func (f *FilterMaps) SetTargetView(targetView *ChainView) {
+ if targetView == nil {
+ panic("nil targetView")
+ }
+ for {
+ select {
+ case <-f.targetViewCh:
+ case f.targetViewCh <- targetView:
+ return
+ }
+ }
+}
+
+// SetFinalBlock sets the finalized block number used for exporting checkpoints.
+// Note that SetFinalBlock never blocks.
+func (f *FilterMaps) SetFinalBlock(finalBlock uint64) {
+ for {
+ select {
+ case <-f.finalBlockCh:
+ case f.finalBlockCh <- finalBlock:
+ return
+ }
+ }
+}
+
+// SetBlockProcessing sets the block processing flag that temporarily suspends
+// log index rendering.
+// Note that SetBlockProcessing never blocks.
+func (f *FilterMaps) SetBlockProcessing(blockProcessing bool) {
+ for {
+ select {
+ case <-f.blockProcessingCh:
+ case f.blockProcessingCh <- blockProcessing:
+ return
+ }
+ }
+}
+
+// WaitIdle blocks until the indexer is in an idle state while synced up to the
+// latest targetView.
+func (f *FilterMaps) WaitIdle() {
+ if f.disabled {
+ f.closeWg.Wait()
+ return
+ }
+ for {
+ ch := make(chan bool)
+ f.waitIdleCh <- ch
+ if <-ch {
+ return
+ }
+ }
+}
+
+// waitForEvent blocks until an event happens that the indexer might react to.
+func (f *FilterMaps) waitForEvent() {
+ for !f.stop && (f.blockProcessing || f.targetHeadIndexed()) {
+ f.processSingleEvent(true)
+ }
+}
+
+// processEvents processes all events, blocking only if a block processing is
+// happening and indexing should be suspended.
+func (f *FilterMaps) processEvents() {
+ for !f.stop && f.processSingleEvent(f.blockProcessing) {
+ }
+}
+
+// processSingleEvent processes a single event either in a blocking or
+// non-blocking manner.
+func (f *FilterMaps) processSingleEvent(blocking bool) bool {
+ if f.matcherSyncRequest != nil && f.targetHeadIndexed() {
+ f.matcherSyncRequest.synced()
+ f.matcherSyncRequest = nil
+ }
+ if blocking {
+ select {
+ case targetView := <-f.targetViewCh:
+ f.setTargetView(targetView)
+ case f.finalBlock = <-f.finalBlockCh:
+ case f.matcherSyncRequest = <-f.matcherSyncCh:
+ case f.blockProcessing = <-f.blockProcessingCh:
+ case <-f.closeCh:
+ f.stop = true
+ case ch := <-f.waitIdleCh:
+ select {
+ case targetView := <-f.targetViewCh:
+ f.setTargetView(targetView)
+ default:
+ }
+ ch <- !f.blockProcessing && f.targetHeadIndexed()
+ }
+ } else {
+ select {
+ case targetView := <-f.targetViewCh:
+ f.setTargetView(targetView)
+ case f.finalBlock = <-f.finalBlockCh:
+ case f.matcherSyncRequest = <-f.matcherSyncCh:
+ case f.blockProcessing = <-f.blockProcessingCh:
+ case <-f.closeCh:
+ f.stop = true
+ default:
+ return false
+ }
+ }
+ return true
+}
+
+// setTargetView updates the target chain view of the iterator.
+func (f *FilterMaps) setTargetView(targetView *ChainView) {
+ f.targetView = targetView
+}
+
+// tryIndexHead tries to render head maps according to the current targetView
+// and returns true if successful.
+func (f *FilterMaps) tryIndexHead() bool {
+ headRenderer, err := f.renderMapsBefore(math.MaxUint32)
+ if err != nil {
+ log.Error("Error creating log index head renderer", "error", err)
+ return false
+ }
+ if headRenderer == nil {
+ return true
+ }
+ if !f.startedHeadIndex {
+ f.lastLogHeadIndex = time.Now()
+ f.startedHeadIndexAt = f.lastLogHeadIndex
+ f.startedHeadIndex = true
+ f.ptrHeadIndex = f.indexedRange.afterLastIndexedBlock
+ }
+ if _, err := headRenderer.run(func() bool {
+ f.processEvents()
+ return f.stop
+ }, func() {
+ f.tryUnindexTail()
+ if f.indexedRange.hasIndexedBlocks() && f.indexedRange.afterLastIndexedBlock >= f.ptrHeadIndex &&
+ ((!f.loggedHeadIndex && time.Since(f.startedHeadIndexAt) > headLogDelay) ||
+ time.Since(f.lastLogHeadIndex) > logFrequency) {
+ log.Info("Log index head rendering in progress",
+ "first block", f.indexedRange.firstIndexedBlock, "last block", f.indexedRange.afterLastIndexedBlock-1,
+ "processed", f.indexedRange.afterLastIndexedBlock-f.ptrHeadIndex,
+ "remaining", f.indexedView.headNumber+1-f.indexedRange.afterLastIndexedBlock,
+ "elapsed", common.PrettyDuration(time.Since(f.startedHeadIndexAt)))
+ f.loggedHeadIndex = true
+ f.lastLogHeadIndex = time.Now()
+ }
+ }); err != nil {
+ log.Error("Log index head rendering failed", "error", err)
+ return false
+ }
+ if f.loggedHeadIndex {
+ log.Info("Log index head rendering finished",
+ "first block", f.indexedRange.firstIndexedBlock, "last block", f.indexedRange.afterLastIndexedBlock-1,
+ "processed", f.indexedRange.afterLastIndexedBlock-f.ptrHeadIndex,
+ "elapsed", common.PrettyDuration(time.Since(f.startedHeadIndexAt)))
+ }
+ f.loggedHeadIndex, f.startedHeadIndex = false, false
+ return true
+}
+
+// tryIndexTail tries to render tail epochs until the tail target block is
+// indexed and returns true if successful.
+// Note that tail indexing is only started if the log index head is fully
+// rendered according to targetView and is suspended as soon as the targetView
+// is changed.
+func (f *FilterMaps) tryIndexTail() bool {
+ for firstEpoch := f.indexedRange.firstRenderedMap >> f.logMapsPerEpoch; firstEpoch > 0 && f.needTailEpoch(firstEpoch-1); {
+ f.processEvents()
+ if f.stop || !f.targetHeadIndexed() {
+ return false
+ }
+ // resume process if tail rendering was interrupted because of head rendering
+ tailRenderer := f.tailRenderer
+ f.tailRenderer = nil
+ if tailRenderer != nil && tailRenderer.afterLastMap != f.indexedRange.firstRenderedMap {
+ tailRenderer = nil
+ }
+ if tailRenderer == nil {
+ var err error
+ tailRenderer, err = f.renderMapsBefore(f.indexedRange.firstRenderedMap)
+ if err != nil {
+ log.Error("Error creating log index tail renderer", "error", err)
+ return false
+ }
+ }
+ if tailRenderer == nil {
+ return true
+ }
+ if !f.startedTailIndex {
+ f.lastLogTailIndex = time.Now()
+ f.startedTailIndexAt = f.lastLogTailIndex
+ f.startedTailIndex = true
+ f.ptrTailIndex = f.indexedRange.firstIndexedBlock - f.tailPartialBlocks()
+ }
+ done, err := tailRenderer.run(func() bool {
+ f.processEvents()
+ return f.stop || !f.targetHeadIndexed()
+ }, func() {
+ tpb, ttb := f.tailPartialBlocks(), f.tailTargetBlock()
+ remaining := uint64(1)
+ if f.indexedRange.firstIndexedBlock > ttb+tpb {
+ remaining = f.indexedRange.firstIndexedBlock - ttb - tpb
+ }
+ if f.indexedRange.hasIndexedBlocks() && f.ptrTailIndex >= f.indexedRange.firstIndexedBlock &&
+ (!f.loggedTailIndex || time.Since(f.lastLogTailIndex) > logFrequency) {
+ log.Info("Log index tail rendering in progress",
+ "first block", f.indexedRange.firstIndexedBlock, "last block", f.indexedRange.afterLastIndexedBlock-1,
+ "processed", f.ptrTailIndex-f.indexedRange.firstIndexedBlock+tpb,
+ "remaining", remaining,
+ "next tail epoch percentage", f.indexedRange.tailPartialEpoch*100/f.mapsPerEpoch,
+ "elapsed", common.PrettyDuration(time.Since(f.startedTailIndexAt)))
+ f.loggedTailIndex = true
+ f.lastLogTailIndex = time.Now()
+ }
+ })
+ if err != nil {
+ log.Error("Log index tail rendering failed", "error", err)
+ }
+ if !done {
+ f.tailRenderer = tailRenderer // only keep tail renderer if interrupted by stopCb
+ return false
+ }
+ }
+ if f.loggedTailIndex {
+ log.Info("Log index tail rendering finished",
+ "first block", f.indexedRange.firstIndexedBlock, "last block", f.indexedRange.afterLastIndexedBlock-1,
+ "processed", f.ptrTailIndex-f.indexedRange.firstIndexedBlock,
+ "elapsed", common.PrettyDuration(time.Since(f.startedTailIndexAt)))
+ f.loggedTailIndex = false
+ }
+ return true
+}
+
+// tryUnindexTail removes entire epochs of log index data as long as the first
+// fully indexed block is at least as old as the tail target.
+// Note that unindexing is very quick as it only removes continuous ranges of
+// data from the database and is also called while running head indexing.
+func (f *FilterMaps) tryUnindexTail() bool {
+ for {
+ firstEpoch := (f.indexedRange.firstRenderedMap - f.indexedRange.tailPartialEpoch) >> f.logMapsPerEpoch
+ if f.needTailEpoch(firstEpoch) {
+ break
+ }
+ f.processEvents()
+ if f.stop {
+ return false
+ }
+ if !f.startedTailUnindex {
+ f.startedTailUnindexAt = time.Now()
+ f.startedTailUnindex = true
+ f.ptrTailUnindexMap = f.indexedRange.firstRenderedMap - f.indexedRange.tailPartialEpoch
+ f.ptrTailUnindexBlock = f.indexedRange.firstIndexedBlock - f.tailPartialBlocks()
+ }
+ if err := f.deleteTailEpoch(firstEpoch); err != nil {
+ log.Error("Log index tail epoch unindexing failed", "error", err)
+ return false
+ }
+ }
+ if f.startedTailUnindex {
+ log.Info("Log index tail unindexing finished",
+ "first block", f.indexedRange.firstIndexedBlock, "last block", f.indexedRange.afterLastIndexedBlock-1,
+ "removed maps", f.indexedRange.firstRenderedMap-f.ptrTailUnindexMap,
+ "removed blocks", f.indexedRange.firstIndexedBlock-f.tailPartialBlocks()-f.ptrTailUnindexBlock,
+ "elapsed", common.PrettyDuration(time.Since(f.startedTailUnindexAt)))
+ f.startedTailUnindex = false
+ }
+ return true
+}
+
+// needTailEpoch returns true if the given tail epoch needs to be kept
+// according to the current tail target, false if it can be removed.
+func (f *FilterMaps) needTailEpoch(epoch uint32) bool {
+ firstEpoch := f.indexedRange.firstRenderedMap >> f.logMapsPerEpoch
+ if epoch > firstEpoch {
+ return true
+ }
+ if epoch+1 < firstEpoch {
+ return false
+ }
+ tailTarget := f.tailTargetBlock()
+ if tailTarget < f.indexedRange.firstIndexedBlock {
+ return true
+ }
+ tailLvIndex, err := f.getBlockLvPointer(tailTarget)
+ if err != nil {
+ log.Error("Could not get log value index of tail block", "error", err)
+ return true
+ }
+ return uint64(epoch+1)<<(f.logValuesPerMap+f.logMapsPerEpoch) >= tailLvIndex
+}
+
+// tailTargetBlock returns the target value for the tail block number according
+// to the log history parameter and the current index head.
+func (f *FilterMaps) tailTargetBlock() uint64 {
+ if f.history == 0 || f.indexedView.headNumber < f.history {
+ return 0
+ }
+ return f.indexedView.headNumber + 1 - f.history
+}
+
+// tailPartialBlocks returns the number of rendered blocks in the partially
+// rendered next tail epoch.
+func (f *FilterMaps) tailPartialBlocks() uint64 {
+ if f.indexedRange.tailPartialEpoch == 0 {
+ return 0
+ }
+ end, _, err := f.getLastBlockOfMap(f.indexedRange.firstRenderedMap - f.mapsPerEpoch + f.indexedRange.tailPartialEpoch - 1)
+ if err != nil {
+ log.Error("Error fetching last block of map", "mapIndex", f.indexedRange.firstRenderedMap-f.mapsPerEpoch+f.indexedRange.tailPartialEpoch-1, "error", err)
+ }
+ var start uint64
+ if f.indexedRange.firstRenderedMap-f.mapsPerEpoch > 0 {
+ start, _, err = f.getLastBlockOfMap(f.indexedRange.firstRenderedMap - f.mapsPerEpoch - 1)
+ if err != nil {
+ log.Error("Error fetching last block of map", "mapIndex", f.indexedRange.firstRenderedMap-f.mapsPerEpoch-1, "error", err)
+ }
+ }
+ return end - start
+}
+
+// targetHeadIndexed returns true if the current log index is consistent with
+// targetView with its head block fully rendered.
+func (f *FilterMaps) targetHeadIndexed() bool {
+ return equalViews(f.targetView, f.indexedView) && f.indexedRange.headBlockIndexed
+}
diff --git a/core/filtermaps/indexer_test.go b/core/filtermaps/indexer_test.go
new file mode 100644
index 00000000000..17ad765e042
--- /dev/null
+++ b/core/filtermaps/indexer_test.go
@@ -0,0 +1,444 @@
+// Copyright 2024 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package filtermaps
+
+import (
+ crand "crypto/rand"
+ "crypto/sha256"
+ "math/big"
+ "math/rand"
+ "sync"
+ "testing"
+
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/consensus/ethash"
+ "github.com/ethereum/go-ethereum/core"
+ "github.com/ethereum/go-ethereum/core/rawdb"
+ "github.com/ethereum/go-ethereum/core/types"
+ "github.com/ethereum/go-ethereum/ethdb"
+ "github.com/ethereum/go-ethereum/params"
+)
+
+var testParams = Params{
+ logMapHeight: 2,
+ logMapWidth: 24,
+ logMapsPerEpoch: 4,
+ logValuesPerMap: 4,
+ baseRowGroupLength: 4,
+ baseRowLengthRatio: 2,
+ logLayerDiff: 2,
+}
+
+func TestIndexerRandomRange(t *testing.T) {
+ ts := newTestSetup(t)
+ defer ts.close()
+
+ forks := make([][]common.Hash, 10)
+ ts.chain.addBlocks(1000, 5, 2, 4, false) // 51 log values per block
+ for i := range forks {
+ if i != 0 {
+ forkBlock := rand.Intn(1000)
+ ts.chain.setHead(forkBlock)
+ ts.chain.addBlocks(1000-forkBlock, 5, 2, 4, false) // 51 log values per block
+ }
+ forks[i] = ts.chain.getCanonicalChain()
+ }
+ lvPerBlock := uint64(51)
+ ts.setHistory(0, false)
+ var (
+ history int
+ noHistory bool
+ fork, head = len(forks) - 1, 1000
+ checkSnapshot bool
+ )
+ ts.fm.WaitIdle()
+ for i := 0; i < 200; i++ {
+ switch rand.Intn(3) {
+ case 0:
+ // change history settings
+ switch rand.Intn(10) {
+ case 0:
+ history, noHistory = 0, false
+ case 1:
+ history, noHistory = 0, true
+ default:
+ history, noHistory = rand.Intn(1000)+1, false
+ }
+ ts.testDisableSnapshots = rand.Intn(2) == 0
+ ts.setHistory(uint64(history), noHistory)
+ case 1:
+ // change head to random position of random fork
+ fork, head = rand.Intn(len(forks)), rand.Intn(1001)
+ ts.chain.setCanonicalChain(forks[fork][:head+1])
+ case 2:
+ if head < 1000 {
+ checkSnapshot = !noHistory && head != 0 // no snapshot generated for block 0
+ // add blocks after the current head
+ head += rand.Intn(1000-head) + 1
+ ts.fm.testSnapshotUsed = false
+ ts.chain.setCanonicalChain(forks[fork][:head+1])
+ }
+ }
+ ts.fm.WaitIdle()
+ if checkSnapshot {
+ if ts.fm.testSnapshotUsed == ts.fm.testDisableSnapshots {
+ ts.t.Fatalf("Invalid snapshot used state after head extension (used: %v, disabled: %v)", ts.fm.testSnapshotUsed, ts.fm.testDisableSnapshots)
+ }
+ checkSnapshot = false
+ }
+ if noHistory {
+ if ts.fm.indexedRange.initialized {
+ t.Fatalf("filterMapsRange initialized while indexing is disabled")
+ }
+ continue
+ }
+ if !ts.fm.indexedRange.initialized {
+ t.Fatalf("filterMapsRange not initialized while indexing is enabled")
+ }
+ var tailBlock uint64
+ if history > 0 && history <= head {
+ tailBlock = uint64(head + 1 - history)
+ }
+ var tailEpoch uint32
+ if tailBlock > 0 {
+ tailLvPtr := (tailBlock - 1) * lvPerBlock // no logs in genesis block, only delimiter
+ tailEpoch = uint32(tailLvPtr >> (testParams.logValuesPerMap + testParams.logMapsPerEpoch))
+ }
+ var expTailBlock uint64
+ if tailEpoch > 0 {
+ tailLvPtr := uint64(tailEpoch) << (testParams.logValuesPerMap + testParams.logMapsPerEpoch) // first available lv ptr
+ // (expTailBlock-1)*lvPerBlock >= tailLvPtr
+ expTailBlock = (tailLvPtr + lvPerBlock*2 - 1) / lvPerBlock
+ }
+ if ts.fm.indexedRange.afterLastIndexedBlock != uint64(head+1) {
+ ts.t.Fatalf("Invalid index head (expected #%d, got #%d)", head, ts.fm.indexedRange.afterLastIndexedBlock-1)
+ }
+ if ts.fm.indexedRange.headBlockDelimiter != uint64(head)*lvPerBlock {
+ ts.t.Fatalf("Invalid index head delimiter pointer (expected %d, got %d)", uint64(head)*lvPerBlock, ts.fm.indexedRange.headBlockDelimiter)
+ }
+ if ts.fm.indexedRange.firstIndexedBlock != expTailBlock {
+ ts.t.Fatalf("Invalid index tail block (expected #%d, got #%d)", expTailBlock, ts.fm.indexedRange.firstIndexedBlock)
+ }
+ }
+}
+
+func TestIndexerCompareDb(t *testing.T) {
+ ts := newTestSetup(t)
+ defer ts.close()
+
+ ts.chain.addBlocks(500, 10, 3, 4, true)
+ ts.setHistory(0, false)
+ ts.fm.WaitIdle()
+ // revert points are stored after block 500
+ ts.chain.addBlocks(500, 10, 3, 4, true)
+ ts.fm.WaitIdle()
+ chain1 := ts.chain.getCanonicalChain()
+ ts.storeDbHash("chain 1 [0, 1000]")
+
+ ts.chain.setHead(600)
+ ts.fm.WaitIdle()
+ ts.storeDbHash("chain 1/2 [0, 600]")
+
+ ts.chain.addBlocks(600, 10, 3, 4, true)
+ ts.fm.WaitIdle()
+ chain2 := ts.chain.getCanonicalChain()
+ ts.storeDbHash("chain 2 [0, 1200]")
+
+ ts.chain.setHead(600)
+ ts.fm.WaitIdle()
+ ts.checkDbHash("chain 1/2 [0, 600]")
+
+ ts.setHistory(800, false)
+ ts.chain.setCanonicalChain(chain1)
+ ts.fm.WaitIdle()
+ ts.storeDbHash("chain 1 [201, 1000]")
+
+ ts.setHistory(0, false)
+ ts.fm.WaitIdle()
+ ts.checkDbHash("chain 1 [0, 1000]")
+
+ ts.setHistory(800, false)
+ ts.chain.setCanonicalChain(chain2)
+ ts.fm.WaitIdle()
+ ts.storeDbHash("chain 2 [401, 1200]")
+
+ ts.setHistory(0, true)
+ ts.fm.WaitIdle()
+ ts.storeDbHash("no index")
+
+ ts.chain.setCanonicalChain(chain2[:501])
+ ts.setHistory(0, false)
+ ts.fm.WaitIdle()
+ ts.chain.setCanonicalChain(chain2)
+ ts.fm.WaitIdle()
+ ts.checkDbHash("chain 2 [0, 1200]")
+
+ ts.chain.setCanonicalChain(chain1)
+ ts.fm.WaitIdle()
+ ts.setHistory(800, false)
+ ts.fm.WaitIdle()
+ ts.checkDbHash("chain 1 [201, 1000]")
+
+ ts.chain.setCanonicalChain(chain2)
+ ts.fm.WaitIdle()
+ ts.checkDbHash("chain 2 [401, 1200]")
+
+ ts.setHistory(0, true)
+ ts.fm.WaitIdle()
+ ts.checkDbHash("no index")
+}
+
+type testSetup struct {
+ t *testing.T
+ fm *FilterMaps
+ db ethdb.Database
+ chain *testChain
+ params Params
+ dbHashes map[string]common.Hash
+ testDisableSnapshots bool
+}
+
+func newTestSetup(t *testing.T) *testSetup {
+ params := testParams
+ params.deriveFields()
+ ts := &testSetup{
+ t: t,
+ db: rawdb.NewMemoryDatabase(),
+ params: params,
+ dbHashes: make(map[string]common.Hash),
+ }
+ ts.chain = ts.newTestChain()
+ return ts
+}
+
+func (ts *testSetup) setHistory(history uint64, noHistory bool) {
+ if ts.fm != nil {
+ ts.fm.Stop()
+ }
+ head := ts.chain.CurrentBlock()
+ view := NewChainView(ts.chain, head.Number.Uint64(), head.Hash())
+ config := Config{
+ History: history,
+ Disabled: noHistory,
+ }
+ ts.fm = NewFilterMaps(ts.db, view, ts.params, config)
+ ts.fm.testDisableSnapshots = ts.testDisableSnapshots
+ ts.fm.Start()
+}
+
+func (ts *testSetup) storeDbHash(id string) {
+ dbHash := ts.fmDbHash()
+ for otherId, otherHash := range ts.dbHashes {
+ if otherHash == dbHash {
+ ts.t.Fatalf("Unexpected equal database hashes `%s` and `%s`", id, otherId)
+ }
+ }
+ ts.dbHashes[id] = dbHash
+}
+
+func (ts *testSetup) checkDbHash(id string) {
+ if ts.fmDbHash() != ts.dbHashes[id] {
+ ts.t.Fatalf("Database `%s` hash mismatch", id)
+ }
+}
+
+func (ts *testSetup) fmDbHash() common.Hash {
+ hasher := sha256.New()
+ it := ts.db.NewIterator(nil, nil)
+ for it.Next() {
+ hasher.Write(it.Key())
+ hasher.Write(it.Value())
+ }
+ it.Release()
+ var result common.Hash
+ hasher.Sum(result[:0])
+ return result
+}
+
+func (ts *testSetup) close() {
+ if ts.fm != nil {
+ ts.fm.Stop()
+ }
+ ts.db.Close()
+ ts.chain.db.Close()
+}
+
+type testChain struct {
+ ts *testSetup
+ db ethdb.Database
+ lock sync.RWMutex
+ canonical []common.Hash
+ blocks map[common.Hash]*types.Block
+ receipts map[common.Hash]types.Receipts
+}
+
+func (ts *testSetup) newTestChain() *testChain {
+ return &testChain{
+ ts: ts,
+ blocks: make(map[common.Hash]*types.Block),
+ receipts: make(map[common.Hash]types.Receipts),
+ }
+}
+
+func (tc *testChain) CurrentBlock() *types.Header {
+ tc.lock.RLock()
+ defer tc.lock.RUnlock()
+
+ if len(tc.canonical) == 0 {
+ return nil
+ }
+ return tc.blocks[tc.canonical[len(tc.canonical)-1]].Header()
+}
+
+func (tc *testChain) GetHeader(hash common.Hash, number uint64) *types.Header {
+ tc.lock.RLock()
+ defer tc.lock.RUnlock()
+
+ if block := tc.blocks[hash]; block != nil {
+ return block.Header()
+ }
+ return nil
+}
+
+func (tc *testChain) GetCanonicalHash(number uint64) common.Hash {
+ tc.lock.RLock()
+ defer tc.lock.RUnlock()
+
+ if uint64(len(tc.canonical)) <= number {
+ return common.Hash{}
+ }
+ return tc.canonical[number]
+}
+
+func (tc *testChain) GetReceiptsByHash(hash common.Hash) types.Receipts {
+ tc.lock.RLock()
+ defer tc.lock.RUnlock()
+
+ return tc.receipts[hash]
+}
+
+func (tc *testChain) addBlocks(count, maxTxPerBlock, maxLogsPerReceipt, maxTopicsPerLog int, random bool) {
+ tc.lock.Lock()
+ blockGen := func(i int, gen *core.BlockGen) {
+ var txCount int
+ if random {
+ txCount = rand.Intn(maxTxPerBlock + 1)
+ } else {
+ txCount = maxTxPerBlock
+ }
+ for k := txCount; k > 0; k-- {
+ receipt := types.NewReceipt(nil, false, 0)
+ var logCount int
+ if random {
+ logCount = rand.Intn(maxLogsPerReceipt + 1)
+ } else {
+ logCount = maxLogsPerReceipt
+ }
+ receipt.Logs = make([]*types.Log, logCount)
+ for i := range receipt.Logs {
+ log := &types.Log{}
+ receipt.Logs[i] = log
+ crand.Read(log.Address[:])
+ var topicCount int
+ if random {
+ topicCount = rand.Intn(maxTopicsPerLog + 1)
+ } else {
+ topicCount = maxTopicsPerLog
+ }
+ log.Topics = make([]common.Hash, topicCount)
+ for j := range log.Topics {
+ crand.Read(log.Topics[j][:])
+ }
+ }
+ gen.AddUncheckedReceipt(receipt)
+ gen.AddUncheckedTx(types.NewTransaction(999, common.HexToAddress("0x999"), big.NewInt(999), 999, gen.BaseFee(), nil))
+ }
+ }
+
+ var (
+ blocks []*types.Block
+ receipts []types.Receipts
+ engine = ethash.NewFaker()
+ )
+
+ if len(tc.canonical) == 0 {
+ gspec := &core.Genesis{
+ Alloc: types.GenesisAlloc{},
+ BaseFee: big.NewInt(params.InitialBaseFee),
+ Config: params.TestChainConfig,
+ }
+ tc.db, blocks, receipts = core.GenerateChainWithGenesis(gspec, engine, count, blockGen)
+ gblock := gspec.ToBlock()
+ ghash := gblock.Hash()
+ tc.canonical = []common.Hash{ghash}
+ tc.blocks[ghash] = gblock
+ tc.receipts[ghash] = types.Receipts{}
+ } else {
+ blocks, receipts = core.GenerateChain(params.TestChainConfig, tc.blocks[tc.canonical[len(tc.canonical)-1]], engine, tc.db, count, blockGen)
+ }
+
+ for i, block := range blocks {
+ num, hash := int(block.NumberU64()), block.Hash()
+ if len(tc.canonical) != num {
+ panic("canonical chain length mismatch")
+ }
+ tc.canonical = append(tc.canonical, hash)
+ tc.blocks[hash] = block
+ if receipts[i] != nil {
+ tc.receipts[hash] = receipts[i]
+ } else {
+ tc.receipts[hash] = types.Receipts{}
+ }
+ }
+ tc.lock.Unlock()
+ tc.setTargetHead()
+}
+
+func (tc *testChain) setHead(headNum int) {
+ tc.lock.Lock()
+ tc.canonical = tc.canonical[:headNum+1]
+ tc.lock.Unlock()
+ tc.setTargetHead()
+}
+
+func (tc *testChain) setTargetHead() {
+ head := tc.CurrentBlock()
+ if tc.ts.fm != nil {
+ if !tc.ts.fm.disabled {
+ //tc.ts.fm.targetViewCh <- NewChainView(tc, head.Number.Uint64(), head.Hash())
+ tc.ts.fm.SetTargetView(NewChainView(tc, head.Number.Uint64(), head.Hash()))
+ }
+ }
+}
+
+func (tc *testChain) getCanonicalChain() []common.Hash {
+ tc.lock.RLock()
+ defer tc.lock.RUnlock()
+
+ cc := make([]common.Hash, len(tc.canonical))
+ copy(cc, tc.canonical)
+ return cc
+}
+
+// restore an earlier state of the chain
+func (tc *testChain) setCanonicalChain(cc []common.Hash) {
+ tc.lock.Lock()
+ tc.canonical = make([]common.Hash, len(cc))
+ copy(tc.canonical, cc)
+ tc.lock.Unlock()
+ tc.setTargetHead()
+}
diff --git a/core/filtermaps/map_renderer.go b/core/filtermaps/map_renderer.go
new file mode 100644
index 00000000000..5bd48a4b23f
--- /dev/null
+++ b/core/filtermaps/map_renderer.go
@@ -0,0 +1,768 @@
+// Copyright 2024 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package filtermaps
+
+import (
+ "errors"
+ "fmt"
+ "math"
+ "sort"
+
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/common/lru"
+ "github.com/ethereum/go-ethereum/core/types"
+ "github.com/ethereum/go-ethereum/log"
+)
+
+const (
+ maxMapsPerBatch = 32 // maximum number of maps rendered in memory
+ valuesPerCallback = 1024 // log values processed per event process callback
+ cachedRowMappings = 10000 // log value to row mappings cached during rendering
+
+ // Number of rows written to db in a single batch.
+ // The map renderer splits up writes like this to ensure that regular
+ // block processing latency is not affected by large batch writes.
+ rowsPerBatch = 1024
+)
+
+var (
+ errChainUpdate = errors.New("rendered section of chain updated")
+)
+
+// mapRenderer represents a process that renders filter maps in a specified
+// range according to the actual targetView.
+type mapRenderer struct {
+ f *FilterMaps
+ afterLastMap uint32
+ currentMap *renderedMap
+ finishedMaps map[uint32]*renderedMap
+ firstFinished, afterLastFinished uint32
+ iterator *logIterator
+}
+
+// renderedMap represents a single filter map that is being rendered in memory.
+type renderedMap struct {
+ filterMap filterMap
+ mapIndex uint32
+ lastBlock uint64
+ lastBlockId common.Hash
+ blockLvPtrs []uint64 // start pointers of blocks starting in this map; last one is lastBlock
+ finished bool // iterator finished; all values rendered
+ headDelimiter uint64 // if finished then points to the future block delimiter of the head block
+}
+
+// firstBlock returns the first block number that starts in the given map.
+func (r *renderedMap) firstBlock() uint64 {
+ return r.lastBlock + 1 - uint64(len(r.blockLvPtrs))
+}
+
+// renderMapsBefore creates a mapRenderer that renders the log index until the
+// specified map index boundary, starting from the latest available starting
+// point that is consistent with the current targetView.
+// The renderer ensures that filterMapsRange, indexedView and the actual map
+// data are always consistent with each other. If afterLastMap is greater than
+// the latest existing rendered map then indexedView is updated to targetView,
+// otherwise it is checked that the rendered range is consistent with both
+// views.
+func (f *FilterMaps) renderMapsBefore(afterLastMap uint32) (*mapRenderer, error) {
+ nextMap, startBlock, startLvPtr, err := f.lastCanonicalMapBoundaryBefore(afterLastMap)
+ if err != nil {
+ return nil, err
+ }
+ if snapshot := f.lastCanonicalSnapshotBefore(afterLastMap); snapshot != nil && snapshot.mapIndex >= nextMap {
+ return f.renderMapsFromSnapshot(snapshot)
+ }
+ if nextMap >= afterLastMap {
+ return nil, nil
+ }
+ return f.renderMapsFromMapBoundary(nextMap, afterLastMap, startBlock, startLvPtr)
+}
+
+// renderMapsFromSnapshot creates a mapRenderer that starts rendering from a
+// snapshot made at a block boundary.
+func (f *FilterMaps) renderMapsFromSnapshot(cp *renderedMap) (*mapRenderer, error) {
+ f.testSnapshotUsed = true
+ iter, err := f.newLogIteratorFromBlockDelimiter(cp.lastBlock)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create log iterator from block delimiter %d: %v", cp.lastBlock, err)
+ }
+ return &mapRenderer{
+ f: f,
+ currentMap: &renderedMap{
+ filterMap: cp.filterMap.copy(),
+ mapIndex: cp.mapIndex,
+ lastBlock: cp.lastBlock,
+ blockLvPtrs: cp.blockLvPtrs,
+ },
+ finishedMaps: make(map[uint32]*renderedMap),
+ firstFinished: cp.mapIndex,
+ afterLastFinished: cp.mapIndex,
+ afterLastMap: math.MaxUint32,
+ iterator: iter,
+ }, nil
+}
+
+// renderMapsFromMapBoundary creates a mapRenderer that starts rendering at a
+// map boundary.
+func (f *FilterMaps) renderMapsFromMapBoundary(firstMap, afterLastMap uint32, startBlock, startLvPtr uint64) (*mapRenderer, error) {
+ iter, err := f.newLogIteratorFromMapBoundary(firstMap, startBlock, startLvPtr)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create log iterator from map boundary %d: %v", firstMap, err)
+ }
+ return &mapRenderer{
+ f: f,
+ currentMap: &renderedMap{
+ filterMap: f.emptyFilterMap(),
+ mapIndex: firstMap,
+ lastBlock: iter.blockNumber,
+ },
+ finishedMaps: make(map[uint32]*renderedMap),
+ firstFinished: firstMap,
+ afterLastFinished: firstMap,
+ afterLastMap: afterLastMap,
+ iterator: iter,
+ }, nil
+}
+
+// lastCanonicalSnapshotBefore returns the latest cached snapshot that matches
+// the current targetView.
+func (f *FilterMaps) lastCanonicalSnapshotBefore(afterLastMap uint32) *renderedMap {
+ var best *renderedMap
+ for _, blockNumber := range f.renderSnapshots.Keys() {
+ if cp, _ := f.renderSnapshots.Get(blockNumber); cp != nil && blockNumber < f.indexedRange.afterLastIndexedBlock &&
+ blockNumber <= f.targetView.headNumber && f.targetView.getBlockId(blockNumber) == cp.lastBlockId &&
+ cp.mapIndex < afterLastMap && (best == nil || blockNumber > best.lastBlock) {
+ best = cp
+ }
+ }
+ return best
+}
+
+// lastCanonicalMapBoundaryBefore returns the latest map boundary before the
+// specified map index that matches the current targetView. This can either
+// be a checkpoint (hardcoded or left from a previously unindexed tail epoch)
+// or the boundary of a currently rendered map.
+// Along with the next map index where the rendering can be started, the number
+// and starting log value pointer of the last block is also returned.
+func (f *FilterMaps) lastCanonicalMapBoundaryBefore(afterLastMap uint32) (nextMap uint32, startBlock, startLvPtr uint64, err error) {
+ if !f.indexedRange.initialized {
+ return 0, 0, 0, nil
+ }
+ mapIndex := afterLastMap
+ for {
+ var ok bool
+ if mapIndex, ok = f.lastMapBoundaryBefore(mapIndex); !ok {
+ return 0, 0, 0, nil
+ }
+ lastBlock, lastBlockId, err := f.getLastBlockOfMap(mapIndex)
+ if err != nil {
+ return 0, 0, 0, fmt.Errorf("failed to retrieve last block of reverse iterated map %d: %v", mapIndex, err)
+ }
+ if lastBlock >= f.indexedView.headNumber || lastBlock >= f.targetView.headNumber ||
+ lastBlockId != f.targetView.getBlockId(lastBlock) {
+ // map is not full or inconsistent with targetView; roll back
+ continue
+ }
+ lvPtr, err := f.getBlockLvPointer(lastBlock)
+ if err != nil {
+ return 0, 0, 0, fmt.Errorf("failed to retrieve log value pointer of last canonical boundary block %d: %v", lastBlock, err)
+ }
+ return mapIndex + 1, lastBlock, lvPtr, nil
+ }
+}
+
+// lastMapBoundaryBefore returns the latest map boundary before the specified
+// map index.
+func (f *FilterMaps) lastMapBoundaryBefore(mapIndex uint32) (uint32, bool) {
+ if !f.indexedRange.initialized || f.indexedRange.afterLastRenderedMap == 0 {
+ return 0, false
+ }
+ if mapIndex > f.indexedRange.afterLastRenderedMap {
+ mapIndex = f.indexedRange.afterLastRenderedMap
+ }
+ if mapIndex > f.indexedRange.firstRenderedMap {
+ return mapIndex - 1, true
+ }
+ if mapIndex+f.mapsPerEpoch > f.indexedRange.firstRenderedMap {
+ if mapIndex > f.indexedRange.firstRenderedMap-f.mapsPerEpoch+f.indexedRange.tailPartialEpoch {
+ mapIndex = f.indexedRange.firstRenderedMap - f.mapsPerEpoch + f.indexedRange.tailPartialEpoch
+ }
+ } else {
+ mapIndex = (mapIndex >> f.logMapsPerEpoch) << f.logMapsPerEpoch
+ }
+ if mapIndex == 0 {
+ return 0, false
+ }
+ return mapIndex - 1, true
+}
+
+// emptyFilterMap returns an empty filter map.
+func (f *FilterMaps) emptyFilterMap() filterMap {
+ return make(filterMap, f.mapHeight)
+}
+
+// loadHeadSnapshot loads the last rendered map from the database and creates
+// a snapshot.
+func (f *FilterMaps) loadHeadSnapshot() error {
+ fm, err := f.getFilterMap(f.indexedRange.afterLastRenderedMap - 1)
+ if err != nil {
+ return fmt.Errorf("failed to load head snapshot map %d: %v", f.indexedRange.afterLastRenderedMap-1, err)
+ }
+ lastBlock, _, err := f.getLastBlockOfMap(f.indexedRange.afterLastRenderedMap - 1)
+ if err != nil {
+ return fmt.Errorf("failed to retrieve last block of head snapshot map %d: %v", f.indexedRange.afterLastRenderedMap-1, err)
+ }
+ var firstBlock uint64
+ if f.indexedRange.afterLastRenderedMap > 1 {
+ prevLastBlock, _, err := f.getLastBlockOfMap(f.indexedRange.afterLastRenderedMap - 2)
+ if err != nil {
+ return fmt.Errorf("failed to retrieve last block of map %d before head snapshot: %v", f.indexedRange.afterLastRenderedMap-2, err)
+ }
+ firstBlock = prevLastBlock + 1
+ }
+ lvPtrs := make([]uint64, lastBlock+1-firstBlock)
+ for i := range lvPtrs {
+ lvPtrs[i], err = f.getBlockLvPointer(firstBlock + uint64(i))
+ if err != nil {
+ return fmt.Errorf("failed to retrieve log value pointer of head snapshot block %d: %v", firstBlock+uint64(i), err)
+ }
+ }
+ f.renderSnapshots.Add(f.indexedRange.afterLastIndexedBlock-1, &renderedMap{
+ filterMap: fm,
+ mapIndex: f.indexedRange.afterLastRenderedMap - 1,
+ lastBlock: f.indexedRange.afterLastIndexedBlock - 1,
+ lastBlockId: f.indexedView.getBlockId(f.indexedRange.afterLastIndexedBlock - 1),
+ blockLvPtrs: lvPtrs,
+ finished: true,
+ headDelimiter: f.indexedRange.headBlockDelimiter,
+ })
+ return nil
+}
+
+// makeSnapshot creates a snapshot of the current state of the rendered map.
+func (r *mapRenderer) makeSnapshot() {
+ r.f.renderSnapshots.Add(r.iterator.blockNumber, &renderedMap{
+ filterMap: r.currentMap.filterMap.copy(),
+ mapIndex: r.currentMap.mapIndex,
+ lastBlock: r.iterator.blockNumber,
+ lastBlockId: r.f.targetView.getBlockId(r.currentMap.lastBlock),
+ blockLvPtrs: r.currentMap.blockLvPtrs,
+ finished: true,
+ headDelimiter: r.iterator.lvIndex,
+ })
+}
+
+// run does the actual map rendering. It periodically calls the stopCb callback
+// and if it returns true the process is interrupted an can be resumed later
+// by calling run again. The writeCb callback is called after new maps have
+// been written to disk and the index range has been updated accordingly.
+func (r *mapRenderer) run(stopCb func() bool, writeCb func()) (bool, error) {
+ for {
+ if done, err := r.renderCurrentMap(stopCb); !done {
+ return done, err // stopped or failed
+ }
+ // map finished
+ r.finishedMaps[r.currentMap.mapIndex] = r.currentMap
+ r.afterLastFinished++
+ if len(r.finishedMaps) >= maxMapsPerBatch || r.afterLastFinished&(r.f.baseRowGroupLength-1) == 0 {
+ if err := r.writeFinishedMaps(stopCb); err != nil {
+ return false, err
+ }
+ writeCb()
+ }
+ if r.afterLastFinished == r.afterLastMap || r.iterator.finished {
+ if err := r.writeFinishedMaps(stopCb); err != nil {
+ return false, err
+ }
+ writeCb()
+ return true, nil
+ }
+ r.currentMap = &renderedMap{
+ filterMap: r.f.emptyFilterMap(),
+ mapIndex: r.afterLastFinished,
+ }
+ }
+}
+
+// renderCurrentMap renders a single map.
+func (r *mapRenderer) renderCurrentMap(stopCb func() bool) (bool, error) {
+ if !r.iterator.updateChainView(r.f.targetView) {
+ return false, errChainUpdate
+ }
+ var waitCnt int
+
+ if r.iterator.lvIndex == 0 {
+ r.currentMap.blockLvPtrs = []uint64{0}
+ }
+ type lvPos struct{ rowIndex, layerIndex uint32 }
+ rowMappingCache := lru.NewCache[common.Hash, lvPos](cachedRowMappings)
+ defer rowMappingCache.Purge()
+
+ for r.iterator.lvIndex < uint64(r.currentMap.mapIndex+1)<= valuesPerCallback {
+ if stopCb() {
+ return false, nil
+ }
+ if !r.iterator.updateChainView(r.f.targetView) {
+ return false, errChainUpdate
+ }
+ waitCnt = 0
+ }
+ r.currentMap.lastBlock = r.iterator.blockNumber
+ if r.iterator.delimiter {
+ r.currentMap.lastBlock++
+ r.currentMap.blockLvPtrs = append(r.currentMap.blockLvPtrs, r.iterator.lvIndex+1)
+ }
+ if logValue := r.iterator.getValueHash(); logValue != (common.Hash{}) {
+ lvp, cached := rowMappingCache.Get(logValue)
+ if !cached {
+ lvp = lvPos{rowIndex: r.f.rowIndex(r.currentMap.mapIndex, 0, logValue)}
+ }
+ for uint32(len(r.currentMap.filterMap[lvp.rowIndex])) >= r.f.maxRowLength(lvp.layerIndex) {
+ lvp.layerIndex++
+ lvp.rowIndex = r.f.rowIndex(r.currentMap.mapIndex, lvp.layerIndex, logValue)
+ cached = false
+ }
+ r.currentMap.filterMap[lvp.rowIndex] = append(r.currentMap.filterMap[lvp.rowIndex], r.f.columnIndex(r.iterator.lvIndex, &logValue))
+ if !cached {
+ rowMappingCache.Add(logValue, lvp)
+ }
+ }
+ if err := r.iterator.next(); err != nil {
+ return false, fmt.Errorf("failed to advance log iterator at %d while rendering map %d: %v", r.iterator.lvIndex, r.currentMap.mapIndex, err)
+ }
+ if !r.f.testDisableSnapshots && r.afterLastMap >= r.f.indexedRange.afterLastRenderedMap &&
+ (r.iterator.delimiter || r.iterator.finished) {
+ r.makeSnapshot()
+ }
+ }
+ if r.iterator.finished {
+ r.currentMap.finished = true
+ r.currentMap.headDelimiter = r.iterator.lvIndex
+ }
+ r.currentMap.lastBlockId = r.f.targetView.getBlockId(r.currentMap.lastBlock)
+ return true, nil
+}
+
+// writeFinishedMaps writes rendered maps to the database and updates
+// filterMapsRange and indexedView accordingly.
+func (r *mapRenderer) writeFinishedMaps(pauseCb func() bool) error {
+ if len(r.finishedMaps) == 0 {
+ return nil
+ }
+ r.f.indexLock.Lock()
+ defer r.f.indexLock.Unlock()
+
+ oldRange := r.f.indexedRange
+ tempRange, err := r.getTempRange()
+ if err != nil {
+ return fmt.Errorf("failed to get temporary rendered range: %v", err)
+ }
+ newRange, err := r.getUpdatedRange()
+ if err != nil {
+ return fmt.Errorf("failed to get updated rendered range: %v", err)
+ }
+ renderedView := r.f.targetView // stopCb callback might still change targetView while writing finished maps
+
+ batch := r.f.db.NewBatch()
+ var writeCnt int
+ checkWriteCnt := func() {
+ writeCnt++
+ if writeCnt == rowsPerBatch {
+ writeCnt = 0
+ if err := batch.Write(); err != nil {
+ log.Crit("Error writing log index update batch", "error", err)
+ }
+ // do not exit while in partially written state but do allow processing
+ // events and pausing while block processing is in progress
+ pauseCb()
+ batch = r.f.db.NewBatch()
+ }
+ }
+
+ r.f.setRange(batch, r.f.indexedView, tempRange)
+ // add or update filter rows
+ for rowIndex := uint32(0); rowIndex < r.f.mapHeight; rowIndex++ {
+ var (
+ mapIndices []uint32
+ rows []FilterRow
+ )
+ for mapIndex := r.firstFinished; mapIndex < r.afterLastFinished; mapIndex++ {
+ row := r.finishedMaps[mapIndex].filterMap[rowIndex]
+ if fm, _ := r.f.filterMapCache.Get(mapIndex); fm != nil && row.Equal(fm[rowIndex]) {
+ continue
+ }
+ mapIndices = append(mapIndices, mapIndex)
+ rows = append(rows, row)
+ }
+ if newRange.afterLastRenderedMap == r.afterLastFinished { // head updated; remove future entries
+ for mapIndex := r.afterLastFinished; mapIndex < oldRange.afterLastRenderedMap; mapIndex++ {
+ if fm, _ := r.f.filterMapCache.Get(mapIndex); fm != nil && len(fm[rowIndex]) == 0 {
+ continue
+ }
+ mapIndices = append(mapIndices, mapIndex)
+ rows = append(rows, nil)
+ }
+ }
+ if err := r.f.storeFilterMapRows(batch, mapIndices, rowIndex, rows); err != nil {
+ return fmt.Errorf("failed to store filter maps %v row %d: %v", mapIndices, rowIndex, err)
+ }
+ checkWriteCnt()
+ }
+ // update filter map cache
+ if newRange.afterLastRenderedMap == r.afterLastFinished {
+ // head updated; cache new head maps and remove future entries
+ for mapIndex := r.firstFinished; mapIndex < r.afterLastFinished; mapIndex++ {
+ r.f.filterMapCache.Add(mapIndex, r.finishedMaps[mapIndex].filterMap)
+ }
+ for mapIndex := r.afterLastFinished; mapIndex < oldRange.afterLastRenderedMap; mapIndex++ {
+ r.f.filterMapCache.Remove(mapIndex)
+ }
+ } else {
+ // head not updated; do not cache maps during tail rendering because we
+ // need head maps to be available in the cache
+ for mapIndex := r.firstFinished; mapIndex < r.afterLastFinished; mapIndex++ {
+ r.f.filterMapCache.Remove(mapIndex)
+ }
+ }
+ // add or update block pointers
+ blockNumber := r.finishedMaps[r.firstFinished].firstBlock()
+ for mapIndex := r.firstFinished; mapIndex < r.afterLastFinished; mapIndex++ {
+ renderedMap := r.finishedMaps[mapIndex]
+ r.f.storeLastBlockOfMap(batch, mapIndex, renderedMap.lastBlock, renderedMap.lastBlockId)
+ checkWriteCnt()
+ if blockNumber != renderedMap.firstBlock() {
+ panic("non-continuous block numbers")
+ }
+ for _, lvPtr := range renderedMap.blockLvPtrs {
+ r.f.storeBlockLvPointer(batch, blockNumber, lvPtr)
+ checkWriteCnt()
+ blockNumber++
+ }
+ }
+ if newRange.afterLastRenderedMap == r.afterLastFinished { // head updated; remove future entries
+ for mapIndex := r.afterLastFinished; mapIndex < oldRange.afterLastRenderedMap; mapIndex++ {
+ r.f.deleteLastBlockOfMap(batch, mapIndex)
+ checkWriteCnt()
+ }
+ for ; blockNumber < oldRange.afterLastIndexedBlock; blockNumber++ {
+ r.f.deleteBlockLvPointer(batch, blockNumber)
+ checkWriteCnt()
+ }
+ }
+ r.finishedMaps = make(map[uint32]*renderedMap)
+ r.firstFinished = r.afterLastFinished
+ r.f.setRange(batch, renderedView, newRange)
+ if err := batch.Write(); err != nil {
+ log.Crit("Error writing log index update batch", "error", err)
+ }
+ return nil
+}
+
+// getTempRange returns a temporary filterMapsRange that is committed to the
+// database while the newly rendered maps are partially written. Writing all
+// processed maps in a single database batch would be a serious hit on db
+// performance so instead safety is ensured by first reverting the valid map
+// range to the unchanged region until all new map data is committed.
+func (r *mapRenderer) getTempRange() (filterMapsRange, error) {
+ tempRange := r.f.indexedRange
+ if err := tempRange.addRenderedRange(r.firstFinished, r.firstFinished, r.afterLastMap, r.f.mapsPerEpoch); err != nil {
+ return filterMapsRange{}, fmt.Errorf("failed to update temporary rendered range: %v", err)
+ }
+ if tempRange.firstRenderedMap != r.f.indexedRange.firstRenderedMap {
+ // first rendered map changed; update first indexed block
+ if tempRange.firstRenderedMap > 0 {
+ lastBlock, _, err := r.f.getLastBlockOfMap(tempRange.firstRenderedMap - 1)
+ if err != nil {
+ return filterMapsRange{}, fmt.Errorf("failed to retrieve last block of map %d before temporary range: %v", tempRange.firstRenderedMap-1, err)
+ }
+ tempRange.firstIndexedBlock = lastBlock + 1
+ } else {
+ tempRange.firstIndexedBlock = 0
+ }
+ }
+ if tempRange.afterLastRenderedMap != r.f.indexedRange.afterLastRenderedMap {
+ // first rendered map changed; update first indexed block
+ if tempRange.afterLastRenderedMap > 0 {
+ lastBlock, _, err := r.f.getLastBlockOfMap(tempRange.afterLastRenderedMap - 1)
+ if err != nil {
+ return filterMapsRange{}, fmt.Errorf("failed to retrieve last block of map %d at the end of temporary range: %v", tempRange.afterLastRenderedMap-1, err)
+ }
+ tempRange.afterLastIndexedBlock = lastBlock
+ } else {
+ tempRange.afterLastIndexedBlock = 0
+ }
+ tempRange.headBlockDelimiter = 0
+ }
+ return tempRange, nil
+}
+
+// getUpdatedRange returns the updated filterMapsRange after writing the newly
+// rendered maps.
+func (r *mapRenderer) getUpdatedRange() (filterMapsRange, error) {
+ // update filterMapsRange
+ newRange := r.f.indexedRange
+ if err := newRange.addRenderedRange(r.firstFinished, r.afterLastFinished, r.afterLastMap, r.f.mapsPerEpoch); err != nil {
+ return filterMapsRange{}, fmt.Errorf("failed to update rendered range: %v", err)
+ }
+ if newRange.firstRenderedMap != r.f.indexedRange.firstRenderedMap {
+ // first rendered map changed; update first indexed block
+ if newRange.firstRenderedMap > 0 {
+ lastBlock, _, err := r.f.getLastBlockOfMap(newRange.firstRenderedMap - 1)
+ if err != nil {
+ return filterMapsRange{}, fmt.Errorf("failed to retrieve last block of map %d before rendered range: %v", newRange.firstRenderedMap-1, err)
+ }
+ newRange.firstIndexedBlock = lastBlock + 1
+ } else {
+ newRange.firstIndexedBlock = 0
+ }
+ }
+ if newRange.afterLastRenderedMap == r.afterLastFinished {
+ // last rendered map changed; update last indexed block and head pointers
+ lm := r.finishedMaps[r.afterLastFinished-1]
+ newRange.headBlockIndexed = lm.finished
+ if lm.finished {
+ newRange.afterLastIndexedBlock = r.f.targetView.headNumber + 1
+ if lm.lastBlock != r.f.targetView.headNumber {
+ panic("map rendering finished but last block != head block")
+ }
+ newRange.headBlockDelimiter = lm.headDelimiter
+ } else {
+ newRange.afterLastIndexedBlock = lm.lastBlock
+ newRange.headBlockDelimiter = 0
+ }
+ } else {
+ // last rendered map not replaced; ensure that target chain view matches
+ // indexed chain view on the rendered section
+ if lastBlock := r.finishedMaps[r.afterLastFinished-1].lastBlock; !matchViews(r.f.indexedView, r.f.targetView, lastBlock) {
+ return filterMapsRange{}, errChainUpdate
+ }
+ }
+ return newRange, nil
+}
+
+// addRenderedRange adds the range [firstRendered, afterLastRendered) and
+// removes [afterLastRendered, afterLastRemoved) from the set of rendered maps.
+func (fmr *filterMapsRange) addRenderedRange(firstRendered, afterLastRendered, afterLastRemoved, mapsPerEpoch uint32) error {
+ if !fmr.initialized {
+ return errors.New("log index not initialized")
+ }
+
+ // Here we create a slice of endpoints for the rendered sections. There are two endpoints
+ // for each section: the index of the first map, and the index after the last map in the
+ // section. We then iterate the endpoints -- adding d values -- to determine whether the
+ // sections are contiguous or whether they have a gap.
+ type endpoint struct {
+ m uint32
+ d int
+ }
+ endpoints := []endpoint{{fmr.firstRenderedMap, 1}, {fmr.afterLastRenderedMap, -1}, {firstRendered, 1}, {afterLastRendered, -101}, {afterLastRemoved, 100}}
+ if fmr.tailPartialEpoch > 0 {
+ endpoints = append(endpoints, []endpoint{{fmr.firstRenderedMap - mapsPerEpoch, 1}, {fmr.firstRenderedMap - mapsPerEpoch + fmr.tailPartialEpoch, -1}}...)
+ }
+ sort.Slice(endpoints, func(i, j int) bool { return endpoints[i].m < endpoints[j].m })
+ var (
+ sum int
+ merged []uint32
+ last bool
+ )
+ for i, e := range endpoints {
+ sum += e.d
+ if i < len(endpoints)-1 && endpoints[i+1].m == e.m {
+ continue
+ }
+ if (sum > 0) != last {
+ merged = append(merged, e.m)
+ last = !last
+ }
+ }
+
+ switch len(merged) {
+ case 0:
+ // Initialized database, but no finished maps yet.
+ fmr.tailPartialEpoch = 0
+ fmr.firstRenderedMap = firstRendered
+ fmr.afterLastRenderedMap = firstRendered
+
+ case 2:
+ // One rendered section (no partial tail epoch).
+ fmr.tailPartialEpoch = 0
+ fmr.firstRenderedMap = merged[0]
+ fmr.afterLastRenderedMap = merged[1]
+
+ case 4:
+ // Two rendered sections (with a gap).
+ // First section (merged[0]-merged[1]) is for the partial tail epoch,
+ // and it has to start exactly one epoch before the main section.
+ if merged[2] != merged[0]+mapsPerEpoch {
+ return fmt.Errorf("invalid tail partial epoch: %v", merged)
+ }
+ fmr.tailPartialEpoch = merged[1] - merged[0]
+ fmr.firstRenderedMap = merged[2]
+ fmr.afterLastRenderedMap = merged[3]
+
+ default:
+ return fmt.Errorf("invalid number of rendered sections: %v", merged)
+ }
+ return nil
+}
+
+// logIterator iterates on the linear log value index range.
+type logIterator struct {
+ chainView *ChainView
+ blockNumber uint64
+ receipts types.Receipts
+ blockStart, delimiter, finished bool
+ txIndex, logIndex, topicIndex int
+ lvIndex uint64
+}
+
+var errUnindexedRange = errors.New("unindexed range")
+
+// newLogIteratorFromBlockDelimiter creates a logIterator starting at the
+// given block's first log value entry (the block delimiter), according to the
+// current targetView.
+func (f *FilterMaps) newLogIteratorFromBlockDelimiter(blockNumber uint64) (*logIterator, error) {
+ if blockNumber > f.targetView.headNumber {
+ return nil, fmt.Errorf("iterator entry point %d after target chain head block %d", blockNumber, f.targetView.headNumber)
+ }
+ if blockNumber < f.indexedRange.firstIndexedBlock || blockNumber >= f.indexedRange.afterLastIndexedBlock {
+ return nil, errUnindexedRange
+ }
+ var lvIndex uint64
+ if f.indexedRange.headBlockIndexed && blockNumber+1 == f.indexedRange.afterLastIndexedBlock {
+ lvIndex = f.indexedRange.headBlockDelimiter
+ } else {
+ var err error
+ lvIndex, err = f.getBlockLvPointer(blockNumber + 1)
+ if err != nil {
+ return nil, fmt.Errorf("failed to retrieve log value pointer of block %d after delimiter: %v", blockNumber+1, err)
+ }
+ lvIndex--
+ }
+ finished := blockNumber == f.targetView.headNumber
+ return &logIterator{
+ chainView: f.targetView,
+ blockNumber: blockNumber,
+ finished: finished,
+ delimiter: !finished,
+ lvIndex: lvIndex,
+ }, nil
+}
+
+// newLogIteratorFromMapBoundary creates a logIterator starting at the given
+// map boundary, according to the current targetView.
+func (f *FilterMaps) newLogIteratorFromMapBoundary(mapIndex uint32, startBlock, startLvPtr uint64) (*logIterator, error) {
+ if startBlock > f.targetView.headNumber {
+ return nil, fmt.Errorf("iterator entry point %d after target chain head block %d", startBlock, f.targetView.headNumber)
+ }
+ // get block receipts
+ receipts := f.targetView.getReceipts(startBlock)
+ if receipts == nil {
+ return nil, fmt.Errorf("receipts not found for start block %d", startBlock)
+ }
+ // initialize iterator at block start
+ l := &logIterator{
+ chainView: f.targetView,
+ blockNumber: startBlock,
+ receipts: receipts,
+ blockStart: true,
+ lvIndex: startLvPtr,
+ }
+ l.nextValid()
+ targetIndex := uint64(mapIndex) << f.logValuesPerMap
+ if l.lvIndex > targetIndex {
+ return nil, fmt.Errorf("log value pointer %d of last block of map is after map boundary %d", l.lvIndex, targetIndex)
+ }
+ // iterate to map boundary
+ for l.lvIndex < targetIndex {
+ if l.finished {
+ return nil, fmt.Errorf("iterator already finished at %d before map boundary target %d", l.lvIndex, targetIndex)
+ }
+ if err := l.next(); err != nil {
+ return nil, fmt.Errorf("failed to advance log iterator at %d before map boundary target %d: %v", l.lvIndex, targetIndex, err)
+ }
+ }
+ return l, nil
+}
+
+// updateChainView updates the iterator's chain view if it still matches the
+// previous view at the current position. Returns true if successful.
+func (l *logIterator) updateChainView(cv *ChainView) bool {
+ if !matchViews(cv, l.chainView, l.blockNumber) {
+ return false
+ }
+ l.chainView = cv
+ return true
+}
+
+// getValueHash returns the log value hash at the current position.
+func (l *logIterator) getValueHash() common.Hash {
+ if l.delimiter || l.finished {
+ return common.Hash{}
+ }
+ log := l.receipts[l.txIndex].Logs[l.logIndex]
+ if l.topicIndex == 0 {
+ return addressValue(log.Address)
+ }
+ return topicValue(log.Topics[l.topicIndex-1])
+}
+
+// next moves the iterator to the next log value index.
+func (l *logIterator) next() error {
+ if l.finished {
+ return nil
+ }
+ if l.delimiter {
+ l.delimiter = false
+ l.blockNumber++
+ l.receipts = l.chainView.getReceipts(l.blockNumber)
+ if l.receipts == nil {
+ return fmt.Errorf("receipts not found for block %d", l.blockNumber)
+ }
+ l.txIndex, l.logIndex, l.topicIndex, l.blockStart = 0, 0, 0, true
+ } else {
+ l.topicIndex++
+ l.blockStart = false
+ }
+ l.lvIndex++
+ l.nextValid()
+ return nil
+}
+
+// nextValid updates the internal transaction, log and topic index pointers
+// to the next existing log value of the given block if necessary.
+// Note that nextValid does not advance the log value index pointer.
+func (l *logIterator) nextValid() {
+ for ; l.txIndex < len(l.receipts); l.txIndex++ {
+ receipt := l.receipts[l.txIndex]
+ for ; l.logIndex < len(receipt.Logs); l.logIndex++ {
+ log := receipt.Logs[l.logIndex]
+ if l.topicIndex <= len(log.Topics) {
+ return
+ }
+ l.topicIndex = 0
+ }
+ l.logIndex = 0
+ }
+ if l.blockNumber == l.chainView.headNumber {
+ l.finished = true
+ } else {
+ l.delimiter = true
+ }
+}
diff --git a/core/filtermaps/matcher.go b/core/filtermaps/matcher.go
new file mode 100644
index 00000000000..19acbd762af
--- /dev/null
+++ b/core/filtermaps/matcher.go
@@ -0,0 +1,934 @@
+// Copyright 2024 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package filtermaps
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "math"
+ "sync"
+ "sync/atomic"
+ "time"
+
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/common/mclock"
+ "github.com/ethereum/go-ethereum/core/types"
+ "github.com/ethereum/go-ethereum/log"
+)
+
+const doRuntimeStats = false
+
+// ErrMatchAll is returned when the specified filter matches everything.
+// Handling this case in filtermaps would require an extra special case and
+// would actually be slower than reverting to legacy filter.
+var ErrMatchAll = errors.New("match all patterns not supported")
+
+// MatcherBackend defines the functions required for searching in the log index
+// data structure. It is currently implemented by FilterMapsMatcherBackend but
+// once EIP-7745 is implemented and active, these functions can also be trustlessly
+// served by a remote prover.
+type MatcherBackend interface {
+ GetParams() *Params
+ GetBlockLvPointer(ctx context.Context, blockNumber uint64) (uint64, error)
+ GetFilterMapRow(ctx context.Context, mapIndex, rowIndex uint32, baseLayerOnly bool) (FilterRow, error)
+ GetLogByLvIndex(ctx context.Context, lvIndex uint64) (*types.Log, error)
+ SyncLogIndex(ctx context.Context) (SyncRange, error)
+ Close()
+}
+
+// SyncRange is returned by MatcherBackend.SyncLogIndex. It contains the latest
+// chain head, the indexed range that is currently consistent with the chain
+// and the valid range that has not been changed and has been consistent with
+// all states of the chain since the previous SyncLogIndex or the creation of
+// the matcher backend.
+type SyncRange struct {
+ HeadNumber uint64
+ // block range where the index has not changed since the last matcher sync
+ // and therefore the set of matches found in this region is guaranteed to
+ // be valid and complete.
+ Valid bool
+ FirstValid, LastValid uint64
+ // block range indexed according to the given chain head.
+ Indexed bool
+ FirstIndexed, LastIndexed uint64
+}
+
+// GetPotentialMatches returns a list of logs that are potential matches for the
+// given filter criteria. If parts of the log index in the searched range are
+// missing or changed during the search process then the resulting logs belonging
+// to that block range might be missing or incorrect.
+// Also note that the returned list may contain false positives.
+func GetPotentialMatches(ctx context.Context, backend MatcherBackend, firstBlock, lastBlock uint64, addresses []common.Address, topics [][]common.Hash) ([]*types.Log, error) {
+ params := backend.GetParams()
+ var getLogStats runtimeStats
+ // find the log value index range to search
+ firstIndex, err := backend.GetBlockLvPointer(ctx, firstBlock)
+ if err != nil {
+ return nil, fmt.Errorf("failed to retrieve log value pointer for first block %d: %v", firstBlock, err)
+ }
+ lastIndex, err := backend.GetBlockLvPointer(ctx, lastBlock+1)
+ if err != nil {
+ return nil, fmt.Errorf("failed to retrieve log value pointer after last block %d: %v", lastBlock, err)
+ }
+ if lastIndex > 0 {
+ lastIndex--
+ }
+ firstMap, lastMap := uint32(firstIndex>>params.logValuesPerMap), uint32(lastIndex>>params.logValuesPerMap)
+ firstEpoch, lastEpoch := firstMap>>params.logMapsPerEpoch, lastMap>>params.logMapsPerEpoch
+
+ // build matcher according to the given filter criteria
+ matchers := make([]matcher, len(topics)+1)
+ // matchAddress signals a match when there is a match for any of the given
+ // addresses.
+ // If the list of addresses is empty then it creates a "wild card" matcher
+ // that signals every index as a potential match.
+ matchAddress := make(matchAny, len(addresses))
+ for i, address := range addresses {
+ matchAddress[i] = &singleMatcher{backend: backend, value: addressValue(address)}
+ }
+ matchers[0] = matchAddress
+ for i, topicList := range topics {
+ // matchTopic signals a match when there is a match for any of the topics
+ // specified for the given position (topicList).
+ // If topicList is empty then it creates a "wild card" matcher that signals
+ // every index as a potential match.
+ matchTopic := make(matchAny, len(topicList))
+ for j, topic := range topicList {
+ matchTopic[j] = &singleMatcher{backend: backend, value: topicValue(topic)}
+ }
+ matchers[i+1] = matchTopic
+ }
+ // matcher is the final sequence matcher that signals a match when all underlying
+ // matchers signal a match for consecutive log value indices.
+ matcher := newMatchSequence(params, matchers)
+
+ // processEpoch returns the potentially matching logs from the given epoch.
+ processEpoch := func(epochIndex uint32) ([]*types.Log, error) {
+ var logs []*types.Log
+ // create a list of map indices to process
+ fm, lm := epochIndex< lastMap {
+ lm = lastMap
+ }
+ //
+ mapIndices := make([]uint32, lm+1-fm)
+ for i := range mapIndices {
+ mapIndices[i] = fm + uint32(i)
+ }
+ // find potential matches
+ matches, err := getAllMatches(ctx, matcher, mapIndices)
+ if err != nil {
+ return logs, err
+ }
+ // get the actual logs located at the matching log value indices
+ var st int
+ getLogStats.setState(&st, stGetLog)
+ defer getLogStats.setState(&st, stNone)
+ for _, m := range matches {
+ if m == nil {
+ return nil, ErrMatchAll
+ }
+ mlogs, err := getLogsFromMatches(ctx, backend, firstIndex, lastIndex, m)
+ if err != nil {
+ return logs, err
+ }
+ logs = append(logs, mlogs...)
+ }
+ getLogStats.addAmount(st, int64(len(logs)))
+ return logs, nil
+ }
+
+ type task struct {
+ epochIndex uint32
+ logs []*types.Log
+ err error
+ done chan struct{}
+ }
+
+ taskCh := make(chan *task)
+ var wg sync.WaitGroup
+ defer func() {
+ close(taskCh)
+ wg.Wait()
+ }()
+
+ worker := func() {
+ for task := range taskCh {
+ if task == nil {
+ break
+ }
+ task.logs, task.err = processEpoch(task.epochIndex)
+ close(task.done)
+ }
+ wg.Done()
+ }
+
+ start := time.Now()
+ for i := 0; i < 4; i++ {
+ wg.Add(1)
+ go worker()
+ }
+
+ var logs []*types.Log
+ // startEpoch is the next task to send whenever a worker can accept it.
+ // waitEpoch is the next task we are waiting for to finish in order to append
+ // results in the correct order.
+ startEpoch, waitEpoch := firstEpoch, firstEpoch
+ tasks := make(map[uint32]*task)
+ tasks[startEpoch] = &task{epochIndex: startEpoch, done: make(chan struct{})}
+ for waitEpoch <= lastEpoch {
+ select {
+ case taskCh <- tasks[startEpoch]:
+ startEpoch++
+ if startEpoch <= lastEpoch {
+ if tasks[startEpoch] == nil {
+ tasks[startEpoch] = &task{epochIndex: startEpoch, done: make(chan struct{})}
+ }
+ }
+ case <-tasks[waitEpoch].done:
+ logs = append(logs, tasks[waitEpoch].logs...)
+ if err := tasks[waitEpoch].err; err != nil {
+ if err == ErrMatchAll {
+ return logs, err
+ }
+ return logs, fmt.Errorf("failed to process log index epoch %d: %v", waitEpoch, err)
+ }
+ delete(tasks, waitEpoch)
+ waitEpoch++
+ if waitEpoch <= lastEpoch {
+ if tasks[waitEpoch] == nil {
+ tasks[waitEpoch] = &task{epochIndex: waitEpoch, done: make(chan struct{})}
+ }
+ }
+ }
+ }
+ if doRuntimeStats {
+ log.Info("Log search finished", "elapsed", time.Since(start))
+ for i, ma := range matchers {
+ for j, m := range ma.(matchAny) {
+ log.Info("Single matcher stats", "matchSequence", i, "matchAny", j)
+ m.(*singleMatcher).stats.print()
+ }
+ }
+ log.Info("Get log stats")
+ getLogStats.print()
+ }
+ return logs, nil
+}
+
+// getLogsFromMatches returns the list of potentially matching logs located at
+// the given list of matching log indices. Matches outside the firstIndex to
+// lastIndex range are not returned.
+func getLogsFromMatches(ctx context.Context, backend MatcherBackend, firstIndex, lastIndex uint64, matches potentialMatches) ([]*types.Log, error) {
+ var logs []*types.Log
+ for _, match := range matches {
+ if match < firstIndex || match > lastIndex {
+ continue
+ }
+ log, err := backend.GetLogByLvIndex(ctx, match)
+ if err != nil {
+ return logs, fmt.Errorf("failed to retrieve log at index %d: %v", match, err)
+ }
+ if log != nil {
+ logs = append(logs, log)
+ }
+ }
+ return logs, nil
+}
+
+// matcher defines a general abstraction for any matcher configuration that
+// can instantiate a matcherInstance.
+type matcher interface {
+ newInstance(mapIndices []uint32) matcherInstance
+}
+
+// matcherInstance defines a general abstraction for a matcher configuration
+// working on a specific set of map indices and eventually returning a list of
+// potentially matching log value indices.
+// Note that processing happens per mapping layer, each call returning a set
+// of results for the maps where the processing has been finished at the given
+// layer. Map indices can also be dropped before a result is returned for them
+// in case the result is no longer interesting. Dropping indices twice or after
+// a result has been returned has no effect. Exactly one matcherResult is
+// returned per requested map index unless dropped.
+type matcherInstance interface {
+ getMatchesForLayer(ctx context.Context, layerIndex uint32) ([]matcherResult, error)
+ dropIndices(mapIndices []uint32)
+}
+
+// matcherResult contains the list of potentially matching log value indices
+// for a given map index.
+type matcherResult struct {
+ mapIndex uint32
+ matches potentialMatches
+}
+
+// getAllMatches creates an instance for a given matcher and set of map indices,
+// iterates through mapping layers and collects all results, then returns all
+// results in the same order as the map indices were specified.
+func getAllMatches(ctx context.Context, matcher matcher, mapIndices []uint32) ([]potentialMatches, error) {
+ instance := matcher.newInstance(mapIndices)
+ resultsMap := make(map[uint32]potentialMatches)
+ for layerIndex := uint32(0); len(resultsMap) < len(mapIndices); layerIndex++ {
+ results, err := instance.getMatchesForLayer(ctx, layerIndex)
+ if err != nil {
+ return nil, err
+ }
+ for _, result := range results {
+ resultsMap[result.mapIndex] = result.matches
+ }
+ }
+ matches := make([]potentialMatches, len(mapIndices))
+ for i, mapIndex := range mapIndices {
+ matches[i] = resultsMap[mapIndex]
+ }
+ return matches, nil
+}
+
+// singleMatcher implements matcher by returning matches for a single log value hash.
+type singleMatcher struct {
+ backend MatcherBackend
+ value common.Hash
+ stats runtimeStats
+}
+
+// singleMatcherInstance is an instance of singleMatcher.
+type singleMatcherInstance struct {
+ *singleMatcher
+ mapIndices []uint32
+ filterRows map[uint32][]FilterRow
+}
+
+// newInstance creates a new instance of singleMatcher.
+func (m *singleMatcher) newInstance(mapIndices []uint32) matcherInstance {
+ filterRows := make(map[uint32][]FilterRow)
+ for _, idx := range mapIndices {
+ filterRows[idx] = []FilterRow{}
+ }
+ copiedIndices := make([]uint32, len(mapIndices))
+ copy(copiedIndices, mapIndices)
+ return &singleMatcherInstance{
+ singleMatcher: m,
+ mapIndices: copiedIndices,
+ filterRows: filterRows,
+ }
+}
+
+// getMatchesForLayer implements matcherInstance.
+func (m *singleMatcherInstance) getMatchesForLayer(ctx context.Context, layerIndex uint32) (results []matcherResult, err error) {
+ var st int
+ m.stats.setState(&st, stOther)
+ params := m.backend.GetParams()
+ maskedMapIndex, rowIndex := uint32(math.MaxUint32), uint32(0)
+ for _, mapIndex := range m.mapIndices {
+ filterRows, ok := m.filterRows[mapIndex]
+ if !ok {
+ continue
+ }
+ if mm := params.maskedMapIndex(mapIndex, layerIndex); mm != maskedMapIndex {
+ // only recalculate rowIndex when necessary
+ maskedMapIndex = mm
+ rowIndex = params.rowIndex(mapIndex, layerIndex, m.value)
+ }
+ if layerIndex == 0 {
+ m.stats.setState(&st, stFetchFirst)
+ } else {
+ m.stats.setState(&st, stFetchMore)
+ }
+ filterRow, err := m.backend.GetFilterMapRow(ctx, mapIndex, rowIndex, layerIndex == 0)
+ if err != nil {
+ m.stats.setState(&st, stNone)
+ return nil, fmt.Errorf("failed to retrieve filter map %d row %d: %v", mapIndex, rowIndex, err)
+ }
+ m.stats.addAmount(st, int64(len(filterRow)))
+ m.stats.setState(&st, stOther)
+ filterRows = append(filterRows, filterRow)
+ if uint32(len(filterRow)) < params.maxRowLength(layerIndex) {
+ m.stats.setState(&st, stProcess)
+ matches := params.potentialMatches(filterRows, mapIndex, m.value)
+ m.stats.addAmount(st, int64(len(matches)))
+ results = append(results, matcherResult{
+ mapIndex: mapIndex,
+ matches: matches,
+ })
+ m.stats.setState(&st, stOther)
+ delete(m.filterRows, mapIndex)
+ } else {
+ m.filterRows[mapIndex] = filterRows
+ }
+ }
+ m.cleanMapIndices()
+ m.stats.setState(&st, stNone)
+ return results, nil
+}
+
+// dropIndices implements matcherInstance.
+func (m *singleMatcherInstance) dropIndices(dropIndices []uint32) {
+ for _, mapIndex := range dropIndices {
+ delete(m.filterRows, mapIndex)
+ }
+ m.cleanMapIndices()
+}
+
+// cleanMapIndices removes map indices from the list if there is no matching
+// filterRows entry because a result has been returned or the index has been
+// dropped.
+func (m *singleMatcherInstance) cleanMapIndices() {
+ var j int
+ for i, mapIndex := range m.mapIndices {
+ if _, ok := m.filterRows[mapIndex]; ok {
+ if i != j {
+ m.mapIndices[j] = mapIndex
+ }
+ j++
+ }
+ }
+ m.mapIndices = m.mapIndices[:j]
+}
+
+// matchAny combinines a set of matchers and returns a match for every position
+// where any of the underlying matchers signaled a match. A zero-length matchAny
+// acts as a "wild card" that signals a potential match at every position.
+type matchAny []matcher
+
+// matchAnyInstance is an instance of matchAny.
+type matchAnyInstance struct {
+ matchAny
+ childInstances []matcherInstance
+ childResults map[uint32]matchAnyResults
+}
+
+// matchAnyResults is used by matchAnyInstance to collect results from all
+// child matchers for a specific map index. Once all results has been received
+// a merged result is returned for the given map and this structure is discarded.
+type matchAnyResults struct {
+ matches []potentialMatches
+ done []bool
+ needMore int
+}
+
+// newInstance creates a new instance of matchAny.
+func (m matchAny) newInstance(mapIndices []uint32) matcherInstance {
+ if len(m) == 1 {
+ return m[0].newInstance(mapIndices)
+ }
+ childResults := make(map[uint32]matchAnyResults)
+ for _, idx := range mapIndices {
+ childResults[idx] = matchAnyResults{
+ matches: make([]potentialMatches, len(m)),
+ done: make([]bool, len(m)),
+ needMore: len(m),
+ }
+ }
+ childInstances := make([]matcherInstance, len(m))
+ for i, matcher := range m {
+ childInstances[i] = matcher.newInstance(mapIndices)
+ }
+ return &matchAnyInstance{
+ matchAny: m,
+ childInstances: childInstances,
+ childResults: childResults,
+ }
+}
+
+// getMatchesForLayer implements matcherInstance.
+func (m *matchAnyInstance) getMatchesForLayer(ctx context.Context, layerIndex uint32) (mergedResults []matcherResult, err error) {
+ if len(m.matchAny) == 0 {
+ // return "wild card" results (potentialMatches(nil) is interpreted as a
+ // potential match at every log value index of the map).
+ mergedResults = make([]matcherResult, len(m.childResults))
+ var i int
+ for mapIndex := range m.childResults {
+ mergedResults[i] = matcherResult{mapIndex: mapIndex, matches: nil}
+ i++
+ }
+ return mergedResults, nil
+ }
+ for i, childInstance := range m.childInstances {
+ results, err := childInstance.getMatchesForLayer(ctx, layerIndex)
+ if err != nil {
+ return nil, fmt.Errorf("failed to evaluate child matcher on layer %d: %v", layerIndex, err)
+ }
+ for _, result := range results {
+ mr, ok := m.childResults[result.mapIndex]
+ if !ok || mr.done[i] {
+ continue
+ }
+ mr.done[i] = true
+ mr.matches[i] = result.matches
+ mr.needMore--
+ if mr.needMore == 0 || result.matches == nil {
+ mergedResults = append(mergedResults, matcherResult{
+ mapIndex: result.mapIndex,
+ matches: mergeResults(mr.matches),
+ })
+ delete(m.childResults, result.mapIndex)
+ } else {
+ m.childResults[result.mapIndex] = mr
+ }
+ }
+ }
+ return mergedResults, nil
+}
+
+// dropIndices implements matcherInstance.
+func (m *matchAnyInstance) dropIndices(dropIndices []uint32) {
+ for _, childInstance := range m.childInstances {
+ childInstance.dropIndices(dropIndices)
+ }
+ for _, mapIndex := range dropIndices {
+ delete(m.childResults, mapIndex)
+ }
+}
+
+// mergeResults merges multiple lists of matches into a single one, preserving
+// ascending order and filtering out any duplicates.
+func mergeResults(results []potentialMatches) potentialMatches {
+ if len(results) == 0 {
+ return nil
+ }
+ var sumLen int
+ for _, res := range results {
+ if res == nil {
+ // nil is a wild card; all indices in map range are potential matches
+ return nil
+ }
+ sumLen += len(res)
+ }
+ merged := make(potentialMatches, 0, sumLen)
+ for {
+ best := -1
+ for i, res := range results {
+ if len(res) == 0 {
+ continue
+ }
+ if best < 0 || res[0] < results[best][0] {
+ best = i
+ }
+ }
+ if best < 0 {
+ return merged
+ }
+ if len(merged) == 0 || results[best][0] > merged[len(merged)-1] {
+ merged = append(merged, results[best][0])
+ }
+ results[best] = results[best][1:]
+ }
+}
+
+// matchSequence combines two matchers, a "base" and a "next" matcher with a
+// positive integer offset so that the resulting matcher signals a match at log
+// value index X when the base matcher returns a match at X and the next matcher
+// gives a match at X+offset. Note that matchSequence can be used recursively to
+// detect any log value sequence.
+type matchSequence struct {
+ params *Params
+ base, next matcher
+ offset uint64
+ statsLock sync.Mutex
+ baseStats, nextStats matchOrderStats
+}
+
+// newInstance creates a new instance of matchSequence.
+func (m *matchSequence) newInstance(mapIndices []uint32) matcherInstance {
+ // determine set of indices to request from next matcher
+ nextIndices := make([]uint32, 0, len(mapIndices)*3/2)
+ needMatched := make(map[uint32]struct{})
+ baseRequested := make(map[uint32]struct{})
+ nextRequested := make(map[uint32]struct{})
+ for _, mapIndex := range mapIndices {
+ needMatched[mapIndex] = struct{}{}
+ baseRequested[mapIndex] = struct{}{}
+ if _, ok := nextRequested[mapIndex]; !ok {
+ nextIndices = append(nextIndices, mapIndex)
+ nextRequested[mapIndex] = struct{}{}
+ }
+ nextIndices = append(nextIndices, mapIndex+1)
+ nextRequested[mapIndex+1] = struct{}{}
+ }
+ return &matchSequenceInstance{
+ matchSequence: m,
+ baseInstance: m.base.newInstance(mapIndices),
+ nextInstance: m.next.newInstance(nextIndices),
+ needMatched: needMatched,
+ baseRequested: baseRequested,
+ nextRequested: nextRequested,
+ baseResults: make(map[uint32]potentialMatches),
+ nextResults: make(map[uint32]potentialMatches),
+ }
+}
+
+// matchOrderStats collects statistics about the evaluating cost and the
+// occurrence of empty result sets from both base and next child matchers.
+// This allows the optimization of the evaluation order by evaluating the
+// child first that is cheaper and/or gives empty results more often and not
+// evaluating the other child in most cases.
+// Note that matchOrderStats is specific to matchSequence and the results are
+// carried over to future instances as the results are mostly useful when
+// evaluating layer zero of each instance. For this reason it should be used
+// in a thread safe way as is may be accessed from multiple worker goroutines.
+type matchOrderStats struct {
+ totalCount, nonEmptyCount, totalCost uint64
+}
+
+// add collects statistics after a child has been evaluated for a certain layer.
+func (ms *matchOrderStats) add(empty bool, layerIndex uint32) {
+ if empty && layerIndex != 0 {
+ // matchers may be evaluated for higher layers after all results have
+ // been returned. Also, empty results are not relevant when previous
+ // layers yielded matches already, so these cases can be ignored.
+ return
+ }
+ ms.totalCount++
+ if !empty {
+ ms.nonEmptyCount++
+ }
+ ms.totalCost += uint64(layerIndex + 1)
+}
+
+// mergeStats merges two sets of matchOrderStats.
+func (ms *matchOrderStats) mergeStats(add matchOrderStats) {
+ ms.totalCount += add.totalCount
+ ms.nonEmptyCount += add.nonEmptyCount
+ ms.totalCost += add.totalCost
+}
+
+// baseFirst returns true if the base child matcher should be evaluated first.
+func (m *matchSequence) baseFirst() bool {
+ m.statsLock.Lock()
+ bf := float64(m.baseStats.totalCost)*float64(m.nextStats.totalCount)+
+ float64(m.baseStats.nonEmptyCount)*float64(m.nextStats.totalCost) <
+ float64(m.baseStats.totalCost)*float64(m.nextStats.nonEmptyCount)+
+ float64(m.nextStats.totalCost)*float64(m.baseStats.totalCount)
+ m.statsLock.Unlock()
+ return bf
+}
+
+// mergeBaseStats merges a set of matchOrderStats into the base matcher stats.
+func (m *matchSequence) mergeBaseStats(stats matchOrderStats) {
+ m.statsLock.Lock()
+ m.baseStats.mergeStats(stats)
+ m.statsLock.Unlock()
+}
+
+// mergeNextStats merges a set of matchOrderStats into the next matcher stats.
+func (m *matchSequence) mergeNextStats(stats matchOrderStats) {
+ m.statsLock.Lock()
+ m.nextStats.mergeStats(stats)
+ m.statsLock.Unlock()
+}
+
+// newMatchSequence creates a recursive sequence matcher from a list of underlying
+// matchers. The resulting matcher signals a match at log value index X when each
+// underlying matcher matchers[i] returns a match at X+i.
+func newMatchSequence(params *Params, matchers []matcher) matcher {
+ if len(matchers) == 0 {
+ panic("zero length sequence matchers are not allowed")
+ }
+ if len(matchers) == 1 {
+ return matchers[0]
+ }
+ return &matchSequence{
+ params: params,
+ base: newMatchSequence(params, matchers[:len(matchers)-1]),
+ next: matchers[len(matchers)-1],
+ offset: uint64(len(matchers) - 1),
+ }
+}
+
+// matchSequenceInstance is an instance of matchSequence.
+type matchSequenceInstance struct {
+ *matchSequence
+ baseInstance, nextInstance matcherInstance
+ baseRequested, nextRequested, needMatched map[uint32]struct{}
+ baseResults, nextResults map[uint32]potentialMatches
+}
+
+// getMatchesForLayer implements matcherInstance.
+func (m *matchSequenceInstance) getMatchesForLayer(ctx context.Context, layerIndex uint32) (matchedResults []matcherResult, err error) {
+ // decide whether to evaluate base or next matcher first
+ baseFirst := m.baseFirst()
+ if baseFirst {
+ if err := m.evalBase(ctx, layerIndex); err != nil {
+ return nil, err
+ }
+ }
+ if err := m.evalNext(ctx, layerIndex); err != nil {
+ return nil, err
+ }
+ if !baseFirst {
+ if err := m.evalBase(ctx, layerIndex); err != nil {
+ return nil, err
+ }
+ }
+ // evaluate and return matched results where possible
+ for mapIndex := range m.needMatched {
+ if _, ok := m.baseRequested[mapIndex]; ok {
+ continue
+ }
+ if _, ok := m.nextRequested[mapIndex]; ok {
+ continue
+ }
+ if _, ok := m.nextRequested[mapIndex+1]; ok {
+ continue
+ }
+ matchedResults = append(matchedResults, matcherResult{
+ mapIndex: mapIndex,
+ matches: m.params.matchResults(mapIndex, m.offset, m.baseResults[mapIndex], m.nextResults[mapIndex], m.nextResults[mapIndex+1]),
+ })
+ delete(m.needMatched, mapIndex)
+ }
+ return matchedResults, nil
+}
+
+// dropIndices implements matcherInstance.
+func (m *matchSequenceInstance) dropIndices(dropIndices []uint32) {
+ for _, mapIndex := range dropIndices {
+ delete(m.needMatched, mapIndex)
+ }
+ var dropBase, dropNext []uint32
+ for _, mapIndex := range dropIndices {
+ if m.dropBase(mapIndex) {
+ dropBase = append(dropBase, mapIndex)
+ }
+ }
+ m.baseInstance.dropIndices(dropBase)
+ for _, mapIndex := range dropIndices {
+ if m.dropNext(mapIndex) {
+ dropNext = append(dropNext, mapIndex)
+ }
+ if m.dropNext(mapIndex + 1) {
+ dropNext = append(dropNext, mapIndex+1)
+ }
+ }
+ m.nextInstance.dropIndices(dropNext)
+}
+
+// evalBase evaluates the base child matcher and drops map indices from the
+// next matcher if possible.
+func (m *matchSequenceInstance) evalBase(ctx context.Context, layerIndex uint32) error {
+ results, err := m.baseInstance.getMatchesForLayer(ctx, layerIndex)
+ if err != nil {
+ return fmt.Errorf("failed to evaluate base matcher on layer %d: %v", layerIndex, err)
+ }
+ var (
+ dropIndices []uint32
+ stats matchOrderStats
+ )
+ for _, r := range results {
+ m.baseResults[r.mapIndex] = r.matches
+ delete(m.baseRequested, r.mapIndex)
+ stats.add(r.matches != nil && len(r.matches) == 0, layerIndex)
+ }
+ m.mergeBaseStats(stats)
+ for _, r := range results {
+ if m.dropNext(r.mapIndex) {
+ dropIndices = append(dropIndices, r.mapIndex)
+ }
+ if m.dropNext(r.mapIndex + 1) {
+ dropIndices = append(dropIndices, r.mapIndex+1)
+ }
+ }
+ if len(dropIndices) > 0 {
+ m.nextInstance.dropIndices(dropIndices)
+ }
+ return nil
+}
+
+// evalNext evaluates the next child matcher and drops map indices from the
+// base matcher if possible.
+func (m *matchSequenceInstance) evalNext(ctx context.Context, layerIndex uint32) error {
+ results, err := m.nextInstance.getMatchesForLayer(ctx, layerIndex)
+ if err != nil {
+ return fmt.Errorf("failed to evaluate next matcher on layer %d: %v", layerIndex, err)
+ }
+ var (
+ dropIndices []uint32
+ stats matchOrderStats
+ )
+ for _, r := range results {
+ m.nextResults[r.mapIndex] = r.matches
+ delete(m.nextRequested, r.mapIndex)
+ stats.add(r.matches != nil && len(r.matches) == 0, layerIndex)
+ }
+ m.mergeNextStats(stats)
+ for _, r := range results {
+ if r.mapIndex > 0 && m.dropBase(r.mapIndex-1) {
+ dropIndices = append(dropIndices, r.mapIndex-1)
+ }
+ if m.dropBase(r.mapIndex) {
+ dropIndices = append(dropIndices, r.mapIndex)
+ }
+ }
+ if len(dropIndices) > 0 {
+ m.baseInstance.dropIndices(dropIndices)
+ }
+ return nil
+}
+
+// dropBase checks whether the given map index can be dropped from the base
+// matcher based on the known results from the next matcher and removes it
+// from the internal requested set and returns true if possible.
+func (m *matchSequenceInstance) dropBase(mapIndex uint32) bool {
+ if _, ok := m.baseRequested[mapIndex]; !ok {
+ return false
+ }
+ if _, ok := m.needMatched[mapIndex]; ok {
+ if next := m.nextResults[mapIndex]; next == nil ||
+ (len(next) > 0 && next[len(next)-1] >= (uint64(mapIndex)< 0 && nextNext[0] < (uint64(mapIndex+1)< 0 && prevBase[len(prevBase)-1]+m.offset >= (uint64(mapIndex)< 0 && base[0]+m.offset < (uint64(mapIndex+1)< 0 {
+ // discard items from nextRes whose corresponding base matcher results
+ // with the negative offset applied would be located at mapIndex-1.
+ start := 0
+ for start < len(nextRes) && nextRes[start] < uint64(mapIndex)< 0 {
+ // discard items from nextNextRes whose corresponding base matcher results
+ // with the negative offset applied would still be located at mapIndex+1.
+ stop := 0
+ for stop < len(nextNextRes) && nextNextRes[stop] < uint64(mapIndex+1)< 0 && len(baseRes) > 0 {
+ if nextRes[0] > baseRes[0]+offset {
+ baseRes = baseRes[1:]
+ } else if nextRes[0] < baseRes[0]+offset {
+ nextRes = nextRes[1:]
+ } else {
+ matchedRes = append(matchedRes, baseRes[0])
+ baseRes = baseRes[1:]
+ nextRes = nextRes[1:]
+ }
+ }
+ } else {
+ // baseRes is a wild card so just return next matcher results with
+ // negative offset.
+ for len(nextRes) > 0 {
+ matchedRes = append(matchedRes, nextRes[0]-offset)
+ nextRes = nextRes[1:]
+ }
+ }
+ }
+ return matchedRes
+}
+
+// runtimeStats collects processing time statistics while searching in the log
+// index. Used only when the doRuntimeStats global flag is true.
+type runtimeStats struct {
+ dt, cnt, amount [stCount]int64
+}
+
+const (
+ stNone = iota
+ stFetchFirst
+ stFetchMore
+ stProcess
+ stGetLog
+ stOther
+ stCount
+)
+
+var stNames = []string{"", "fetchFirst", "fetchMore", "process", "getLog", "other"}
+
+// set sets the processing state to one of the pre-defined constants.
+// Processing time spent in each state is measured separately.
+func (ts *runtimeStats) setState(state *int, newState int) {
+ if !doRuntimeStats || newState == *state {
+ return
+ }
+ now := int64(mclock.Now())
+ atomic.AddInt64(&ts.dt[*state], now)
+ atomic.AddInt64(&ts.dt[newState], -now)
+ atomic.AddInt64(&ts.cnt[newState], 1)
+ *state = newState
+}
+
+func (ts *runtimeStats) addAmount(state int, amount int64) {
+ atomic.AddInt64(&ts.amount[state], amount)
+}
+
+// print prints the collected statistics.
+func (ts *runtimeStats) print() {
+ for i := 1; i < stCount; i++ {
+ log.Info("Matcher stats", "name", stNames[i], "dt", time.Duration(ts.dt[i]), "count", ts.cnt[i], "amount", ts.amount[i])
+ }
+}
diff --git a/core/filtermaps/matcher_backend.go b/core/filtermaps/matcher_backend.go
new file mode 100644
index 00000000000..66f95f21459
--- /dev/null
+++ b/core/filtermaps/matcher_backend.go
@@ -0,0 +1,206 @@
+// Copyright 2024 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package filtermaps
+
+import (
+ "context"
+
+ "github.com/ethereum/go-ethereum/core/types"
+)
+
+// FilterMapsMatcherBackend implements MatcherBackend.
+type FilterMapsMatcherBackend struct {
+ f *FilterMaps
+
+ // these fields should be accessed under f.matchersLock mutex.
+ valid bool
+ firstValid, lastValid uint64
+ syncCh chan SyncRange
+}
+
+// NewMatcherBackend returns a FilterMapsMatcherBackend after registering it in
+// the active matcher set.
+// Note that Close should always be called when the matcher is no longer used.
+func (f *FilterMaps) NewMatcherBackend() *FilterMapsMatcherBackend {
+ f.indexLock.RLock()
+ f.matchersLock.Lock()
+ defer func() {
+ f.matchersLock.Unlock()
+ f.indexLock.RUnlock()
+ }()
+
+ fm := &FilterMapsMatcherBackend{
+ f: f,
+ valid: f.indexedRange.initialized && f.indexedRange.afterLastIndexedBlock > f.indexedRange.firstIndexedBlock,
+ firstValid: f.indexedRange.firstIndexedBlock,
+ lastValid: f.indexedRange.afterLastIndexedBlock - 1,
+ }
+ f.matchers[fm] = struct{}{}
+ return fm
+}
+
+// GetParams returns the filtermaps parameters.
+// GetParams implements MatcherBackend.
+func (fm *FilterMapsMatcherBackend) GetParams() *Params {
+ return &fm.f.Params
+}
+
+// Close removes the matcher from the set of active matchers and ensures that
+// any SyncLogIndex calls are cancelled.
+// Close implements MatcherBackend.
+func (fm *FilterMapsMatcherBackend) Close() {
+ fm.f.matchersLock.Lock()
+ defer fm.f.matchersLock.Unlock()
+
+ delete(fm.f.matchers, fm)
+}
+
+// GetFilterMapRow returns the given row of the given map. If the row is empty
+// then a non-nil zero length row is returned. If baseLayerOnly is true then
+// only the first baseRowLength entries of the row are guaranteed to be
+// returned.
+// Note that the returned slices should not be modified, they should be copied
+// on write.
+// GetFilterMapRow implements MatcherBackend.
+func (fm *FilterMapsMatcherBackend) GetFilterMapRow(ctx context.Context, mapIndex, rowIndex uint32, baseLayerOnly bool) (FilterRow, error) {
+ return fm.f.getFilterMapRow(mapIndex, rowIndex, baseLayerOnly)
+}
+
+// GetBlockLvPointer returns the starting log value index where the log values
+// generated by the given block are located. If blockNumber is beyond the current
+// head then the first unoccupied log value index is returned.
+// GetBlockLvPointer implements MatcherBackend.
+func (fm *FilterMapsMatcherBackend) GetBlockLvPointer(ctx context.Context, blockNumber uint64) (uint64, error) {
+ fm.f.indexLock.RLock()
+ defer fm.f.indexLock.RUnlock()
+
+ return fm.f.getBlockLvPointer(blockNumber)
+}
+
+// GetLogByLvIndex returns the log at the given log value index.
+// Note that this function assumes that the log index structure is consistent
+// with the canonical chain at the point where the given log value index points.
+// If this is not the case then an invalid result may be returned or certain
+// logs might not be returned at all.
+// No error is returned though because of an inconsistency between the chain and
+// the log index. It is the caller's responsibility to verify this consistency
+// using SyncLogIndex and re-process certain blocks if necessary.
+// GetLogByLvIndex implements MatcherBackend.
+func (fm *FilterMapsMatcherBackend) GetLogByLvIndex(ctx context.Context, lvIndex uint64) (*types.Log, error) {
+ fm.f.indexLock.RLock()
+ defer fm.f.indexLock.RUnlock()
+
+ return fm.f.getLogByLvIndex(lvIndex)
+}
+
+// synced signals to the matcher that has triggered a synchronisation that it
+// has been finished and the log index is consistent with the chain head passed
+// as a parameter.
+// Note that if the log index head was far behind the chain head then it might not
+// be synced up to the given head in a single step. Still, the latest chain head
+// should be passed as a parameter and the existing log index should be consistent
+// with that chain.
+func (fm *FilterMapsMatcherBackend) synced() {
+ fm.f.indexLock.RLock()
+ fm.f.matchersLock.Lock()
+ defer func() {
+ fm.f.matchersLock.Unlock()
+ fm.f.indexLock.RUnlock()
+ }()
+
+ var (
+ indexed bool
+ lastIndexed, subLastIndexed uint64
+ )
+ if !fm.f.indexedRange.headBlockIndexed {
+ subLastIndexed = 1
+ }
+ if fm.f.indexedRange.afterLastIndexedBlock-subLastIndexed > fm.f.indexedRange.firstIndexedBlock {
+ indexed, lastIndexed = true, fm.f.indexedRange.afterLastIndexedBlock-subLastIndexed-1
+ }
+ fm.syncCh <- SyncRange{
+ HeadNumber: fm.f.indexedView.headNumber,
+ Valid: fm.valid,
+ FirstValid: fm.firstValid,
+ LastValid: fm.lastValid,
+ Indexed: indexed,
+ FirstIndexed: fm.f.indexedRange.firstIndexedBlock,
+ LastIndexed: lastIndexed,
+ }
+ fm.valid = indexed
+ fm.firstValid = fm.f.indexedRange.firstIndexedBlock
+ fm.lastValid = lastIndexed
+ fm.syncCh = nil
+}
+
+// SyncLogIndex ensures that the log index is consistent with the current state
+// of the chain and is synced up to the current head. It blocks until this state
+// is achieved or the context is cancelled.
+// If successful, it returns a SyncRange that contains the latest chain head,
+// the indexed range that is currently consistent with the chain and the valid
+// range that has not been changed and has been consistent with all states of the
+// chain since the previous SyncLogIndex or the creation of the matcher backend.
+func (fm *FilterMapsMatcherBackend) SyncLogIndex(ctx context.Context) (SyncRange, error) {
+ if fm.f.disabled {
+ return SyncRange{HeadNumber: fm.f.targetView.headNumber}, nil
+ }
+
+ syncCh := make(chan SyncRange, 1)
+ fm.f.matchersLock.Lock()
+ fm.syncCh = syncCh
+ fm.f.matchersLock.Unlock()
+
+ select {
+ case fm.f.matcherSyncCh <- fm:
+ case <-ctx.Done():
+ return SyncRange{}, ctx.Err()
+ }
+ select {
+ case vr := <-syncCh:
+ return vr, nil
+ case <-ctx.Done():
+ return SyncRange{}, ctx.Err()
+ }
+}
+
+// updateMatchersValidRange iterates through active matchers and limits their
+// valid range with the current indexed range. This function should be called
+// whenever a part of the log index has been removed, before adding new blocks
+// to it.
+// Note that this function assumes that the index read lock is being held.
+func (f *FilterMaps) updateMatchersValidRange() {
+ f.matchersLock.Lock()
+ defer f.matchersLock.Unlock()
+
+ for fm := range f.matchers {
+ if !f.indexedRange.hasIndexedBlocks() {
+ fm.valid = false
+ }
+ if !fm.valid {
+ continue
+ }
+ if fm.firstValid < f.indexedRange.firstIndexedBlock {
+ fm.firstValid = f.indexedRange.firstIndexedBlock
+ }
+ if fm.lastValid >= f.indexedRange.afterLastIndexedBlock {
+ fm.lastValid = f.indexedRange.afterLastIndexedBlock - 1
+ }
+ if fm.firstValid > fm.lastValid {
+ fm.valid = false
+ }
+ }
+}
diff --git a/core/filtermaps/matcher_test.go b/core/filtermaps/matcher_test.go
new file mode 100644
index 00000000000..5a49ce66ecb
--- /dev/null
+++ b/core/filtermaps/matcher_test.go
@@ -0,0 +1,87 @@
+// Copyright 2024 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package filtermaps
+
+import (
+ "context"
+ crand "crypto/rand"
+ "math/rand"
+ "testing"
+
+ "github.com/ethereum/go-ethereum/common"
+)
+
+func TestMatcher(t *testing.T) {
+ ts := newTestSetup(t)
+ defer ts.close()
+
+ ts.chain.addBlocks(100, 10, 10, 4, true)
+ ts.setHistory(0, false)
+ ts.fm.WaitIdle()
+
+ for i := 0; i < 2000; i++ {
+ bhash := ts.chain.canonical[rand.Intn(len(ts.chain.canonical))]
+ receipts := ts.chain.receipts[bhash]
+ if len(receipts) == 0 {
+ continue
+ }
+ receipt := receipts[rand.Intn(len(receipts))]
+ if len(receipt.Logs) == 0 {
+ continue
+ }
+ log := receipt.Logs[rand.Intn(len(receipt.Logs))]
+ var ok bool
+ addresses := make([]common.Address, rand.Intn(3))
+ for i := range addresses {
+ crand.Read(addresses[i][:])
+ }
+ if len(addresses) > 0 {
+ addresses[rand.Intn(len(addresses))] = log.Address
+ ok = true
+ }
+ topics := make([][]common.Hash, rand.Intn(len(log.Topics)+1))
+ for j := range topics {
+ topics[j] = make([]common.Hash, rand.Intn(3))
+ for i := range topics[j] {
+ crand.Read(topics[j][i][:])
+ }
+ if len(topics[j]) > 0 {
+ topics[j][rand.Intn(len(topics[j]))] = log.Topics[j]
+ ok = true
+ }
+ }
+ if !ok {
+ continue // cannot search for match-all pattern
+ }
+ mb := ts.fm.NewMatcherBackend()
+ logs, err := GetPotentialMatches(context.Background(), mb, 0, 1000, addresses, topics)
+ mb.Close()
+ if err != nil {
+ t.Fatalf("Log search error: %v", err)
+ }
+ var found bool
+ for _, l := range logs {
+ if l == log {
+ found = true
+ break
+ }
+ }
+ if !found {
+ t.Fatalf("Log search did not return expected log (addresses: %v, topics: %v, expected log: %v)", addresses, topics, *log)
+ }
+ }
+}
diff --git a/core/filtermaps/math.go b/core/filtermaps/math.go
new file mode 100644
index 00000000000..762a99be9e2
--- /dev/null
+++ b/core/filtermaps/math.go
@@ -0,0 +1,212 @@
+// Copyright 2024 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package filtermaps
+
+import (
+ "crypto/sha256"
+ "encoding/binary"
+ "hash/fnv"
+ "math"
+ "sort"
+
+ "github.com/ethereum/go-ethereum/common"
+)
+
+// Params defines the basic parameters of the log index structure.
+type Params struct {
+ logMapHeight uint // log2(mapHeight)
+ logMapWidth uint // log2(mapWidth)
+ logMapsPerEpoch uint // log2(mapsPerEpoch)
+ logValuesPerMap uint // log2(logValuesPerMap)
+ baseRowLengthRatio uint // baseRowLength / average row length
+ logLayerDiff uint // maxRowLength log2 growth per layer
+ // derived fields
+ mapHeight uint32 // filter map height (number of rows)
+ mapsPerEpoch uint32 // number of maps in an epoch
+ baseRowLength uint32 // maximum number of log values per row on layer 0
+ valuesPerMap uint64 // number of log values marked on each filter map
+ // not affecting consensus
+ baseRowGroupLength uint32 // length of base row groups in local database
+}
+
+// DefaultParams is the set of parameters used on mainnet.
+var DefaultParams = Params{
+ logMapHeight: 16,
+ logMapWidth: 24,
+ logMapsPerEpoch: 10,
+ logValuesPerMap: 16,
+ baseRowGroupLength: 32,
+ baseRowLengthRatio: 8,
+ logLayerDiff: 4,
+}
+
+// RangeTestParams puts one log value per epoch, ensuring block exact tail unindexing for testing
+var RangeTestParams = Params{
+ logMapHeight: 4,
+ logMapWidth: 24,
+ logMapsPerEpoch: 0,
+ logValuesPerMap: 0,
+ baseRowGroupLength: 32,
+ baseRowLengthRatio: 16, // baseRowLength >= 1
+ logLayerDiff: 4,
+}
+
+// deriveFields calculates the derived fields of the parameter set.
+func (p *Params) deriveFields() {
+ p.mapHeight = uint32(1) << p.logMapHeight
+ p.mapsPerEpoch = uint32(1) << p.logMapsPerEpoch
+ p.valuesPerMap = uint64(1) << p.logValuesPerMap
+ p.baseRowLength = uint32(p.valuesPerMap * uint64(p.baseRowLengthRatio) / uint64(p.mapHeight))
+}
+
+// addressValue returns the log value hash of a log emitting address.
+func addressValue(address common.Address) common.Hash {
+ var result common.Hash
+ hasher := sha256.New()
+ hasher.Write(address[:])
+ hasher.Sum(result[:0])
+ return result
+}
+
+// topicValue returns the log value hash of a log topic.
+func topicValue(topic common.Hash) common.Hash {
+ var result common.Hash
+ hasher := sha256.New()
+ hasher.Write(topic[:])
+ hasher.Sum(result[:0])
+ return result
+}
+
+// rowIndex returns the row index in which the given log value should be marked
+// on the given map and mapping layer. Note that row assignments are re-shuffled
+// with a different frequency on each mapping layer, allowing efficient disk
+// access and Merkle proofs for long sections of short rows on lower order
+// layers while avoiding putting too many heavy rows next to each other on
+// higher order layers.
+func (p *Params) rowIndex(mapIndex, layerIndex uint32, logValue common.Hash) uint32 {
+ hasher := sha256.New()
+ hasher.Write(logValue[:])
+ var indexEnc [8]byte
+ binary.LittleEndian.PutUint32(indexEnc[0:4], p.maskedMapIndex(mapIndex, layerIndex))
+ binary.LittleEndian.PutUint32(indexEnc[4:8], layerIndex)
+ hasher.Write(indexEnc[:])
+ var hash common.Hash
+ hasher.Sum(hash[:0])
+ return binary.LittleEndian.Uint32(hash[:4]) % p.mapHeight
+}
+
+// columnIndex returns the column index where the given log value at the given
+// position should be marked.
+func (p *Params) columnIndex(lvIndex uint64, logValue *common.Hash) uint32 {
+ var indexEnc [8]byte
+ binary.LittleEndian.PutUint64(indexEnc[:], lvIndex)
+ // Note: reusing the hasher brings practically no performance gain and would
+ // require passing it through the entire matcher logic because of multi-thread
+ // matching
+ hasher := fnv.New64a()
+ hasher.Write(indexEnc[:])
+ hasher.Write(logValue[:])
+ hash := hasher.Sum64()
+ hashBits := p.logMapWidth - p.logValuesPerMap
+ return uint32(lvIndex%p.valuesPerMap)<>(64-hashBits)) ^ uint32(hash)>>(32-hashBits))
+}
+
+// maxRowLength returns the maximum length filter rows are populated up to
+// when using the given mapping layer. A log value can be marked on the map
+// according to a given mapping layer if the row mapping on that layer points
+// to a row that has not yet reached the maxRowLength belonging to that layer.
+// This means that a row that is considered full on a given layer may still be
+// extended further on a higher order layer.
+// Each value is marked on the lowest order layer possible, assuming that marks
+// are added in ascending log value index order.
+// When searching for a log value one should consider all layers and process
+// corresponding rows up until the first one where the row mapped to the given
+// layer is not full.
+func (p *Params) maxRowLength(layerIndex uint32) uint32 {
+ logLayerDiff := uint(layerIndex) * p.logLayerDiff
+ if logLayerDiff > p.logMapsPerEpoch {
+ logLayerDiff = p.logMapsPerEpoch
+ }
+ return p.baseRowLength << logLayerDiff
+}
+
+// maskedMapIndex returns the index used for row mapping calculation on the
+// given layer. On layer zero the mapping changes once per epoch, then the
+// frequency of re-mapping increases with every new layer until it reaches
+// the frequency where it is different for every mapIndex.
+func (p *Params) maskedMapIndex(mapIndex, layerIndex uint32) uint32 {
+ logLayerDiff := uint(layerIndex) * p.logLayerDiff
+ if logLayerDiff > p.logMapsPerEpoch {
+ logLayerDiff = p.logMapsPerEpoch
+ }
+ return mapIndex & (uint32(math.MaxUint32) << (p.logMapsPerEpoch - logLayerDiff))
+}
+
+// potentialMatches returns the list of log value indices potentially matching
+// the given log value hash in the range of the filter map the row belongs to.
+// Note that the list of indices is always sorted and potential duplicates are
+// removed. Though the column indices are stored in the same order they were
+// added and therefore the true matches are automatically reverse transformed
+// in the right order, false positives can ruin this property. Since these can
+// only be separated from true matches after the combined pattern matching of the
+// outputs of individual log value matchers and this pattern matcher assumes a
+// sorted and duplicate-free list of indices, we should ensure these properties
+// here.
+func (p *Params) potentialMatches(rows []FilterRow, mapIndex uint32, logValue common.Hash) potentialMatches {
+ results := make(potentialMatches, 0, 8)
+ mapFirst := uint64(mapIndex) << p.logValuesPerMap
+ for i, row := range rows {
+ rowLen, maxLen := len(row), int(p.maxRowLength(uint32(i)))
+ if rowLen > maxLen {
+ rowLen = maxLen // any additional entries are generated by another log value on a higher mapping layer
+ }
+ for i := 0; i < rowLen; i++ {
+ if potentialMatch := mapFirst + uint64(row[i]>>(p.logMapWidth-p.logValuesPerMap)); row[i] == p.columnIndex(potentialMatch, &logValue) {
+ results = append(results, potentialMatch)
+ }
+ }
+ if rowLen < maxLen {
+ break
+ }
+ if i == len(rows)-1 {
+ panic("potentialMatches: insufficient list of row alternatives")
+ }
+ }
+ sort.Sort(results)
+ // remove duplicates
+ j := 0
+ for i, match := range results {
+ if i == 0 || match != results[i-1] {
+ results[j] = results[i]
+ j++
+ }
+ }
+ return results[:j]
+}
+
+// potentialMatches is a strictly monotonically increasing list of log value
+// indices in the range of a filter map that are potential matches for certain
+// filter criteria.
+// potentialMatches implements sort.Interface.
+// Note that nil is used as a wildcard and therefore means that all log value
+// indices in the filter map range are potential matches. If there are no
+// potential matches in the given map's range then an empty slice should be used.
+type potentialMatches []uint64
+
+func (p potentialMatches) Len() int { return len(p) }
+func (p potentialMatches) Less(i, j int) bool { return p[i] < p[j] }
+func (p potentialMatches) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
diff --git a/core/filtermaps/math_test.go b/core/filtermaps/math_test.go
new file mode 100644
index 00000000000..a4c16090598
--- /dev/null
+++ b/core/filtermaps/math_test.go
@@ -0,0 +1,149 @@
+// Copyright 2024 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package filtermaps
+
+import (
+ crand "crypto/rand"
+ "math/rand"
+ "testing"
+
+ "github.com/ethereum/go-ethereum/common"
+)
+
+func TestSingleMatch(t *testing.T) {
+ params := DefaultParams
+ params.deriveFields()
+
+ for count := 0; count < 100000; count++ {
+ // generate a row with a single random entry
+ mapIndex := rand.Uint32()
+ lvIndex := uint64(mapIndex)< 0; i-- {
+ j := rand.Intn(i)
+ row[i], row[j] = row[j], row[i]
+ }
+ // split up into a list of rows if longer than allowed
+ var rows []FilterRow
+ for layerIndex := uint32(0); row != nil; layerIndex++ {
+ maxLen := int(params.maxRowLength(layerIndex))
+ if len(row) > maxLen {
+ rows = append(rows, row[:maxLen])
+ row = row[maxLen:]
+ } else {
+ rows = append(rows, row)
+ row = nil
+ }
+ }
+ // check retrieved matches while also counting false positives
+ for i, lvHash := range lvHashes {
+ matches := params.potentialMatches(rows, mapIndex, lvHash)
+ if i < testPmLen {
+ // check single entry match
+ if len(matches) < 1 {
+ t.Fatalf("Invalid length of matches (got %d, expected >=1)", len(matches))
+ }
+ var found bool
+ for _, lvi := range matches {
+ if lvi == lvIndices[i] {
+ found = true
+ } else {
+ falsePositives++
+ }
+ }
+ if !found {
+ t.Fatalf("Expected match not found (got %v, expected %d)", matches, lvIndices[i])
+ }
+ } else {
+ // check "long series" match
+ if len(matches) < testPmLen {
+ t.Fatalf("Invalid length of matches (got %d, expected >=%d)", len(matches), testPmLen)
+ }
+ // since results are ordered, first testPmLen entries should always match exactly
+ for j := 0; j < testPmLen; j++ {
+ if matches[j] != lvStart+uint64(j) {
+ t.Fatalf("Incorrect match at index %d (got %d, expected %d)", j, matches[j], lvStart+uint64(j))
+ }
+ }
+ // the rest are false positives
+ falsePositives += len(matches) - testPmLen
+ }
+ }
+ }
+ // Whenever looking for a certain log value hash, each entry in the row that
+ // was generated by another log value hash (a "foreign entry") has a
+ // valuesPerMap // 2^32 chance of yielding a false positive if the reverse
+ // transformed 32 bit integer is by random chance less than valuesPerMap and
+ // is therefore considered a potentially valid match.
+ // We have testPmLen unique hash entries and a testPmLen long series of entries
+ // for the same hash. For each of the testPmLen unique hash entries there are
+ // testPmLen*2-1 foreign entries while for the long series there are testPmLen
+ // foreign entries. This means that after performing all these filtering runs,
+ // we have processed 2*testPmLen^2 foreign entries, which given us an estimate
+ // of how many false positives to expect.
+ expFalse := int(uint64(testPmCount*testPmLen*testPmLen*2) * params.valuesPerMap >> params.logMapWidth)
+ if falsePositives < expFalse/2 || falsePositives > expFalse*3/2 {
+ t.Fatalf("False positive rate out of expected range (got %d, expected %d +-50%%)", falsePositives, expFalse)
+ }
+}
diff --git a/core/rawdb/accessors_indexes.go b/core/rawdb/accessors_indexes.go
index 4f2ef0a8808..f679c3aeb39 100644
--- a/core/rawdb/accessors_indexes.go
+++ b/core/rawdb/accessors_indexes.go
@@ -18,6 +18,8 @@ package rawdb
import (
"bytes"
+ "encoding/binary"
+ "errors"
"math/big"
"github.com/ethereum/go-ethereum/common"
@@ -179,3 +181,306 @@ func DeleteBloombits(db ethdb.Database, bit uint, from uint64, to uint64) {
log.Crit("Failed to delete bloom bits", "err", it.Error())
}
}
+
+// ReadFilterMapRow retrieves a filter map row at the given mapRowIndex
+// (see filtermaps.mapRowIndex for the storage index encoding).
+// Note that zero length rows are not stored in the database and therefore all
+// non-existent entries are interpreted as empty rows and return no error.
+// Also note that the mapRowIndex indexing scheme is the same as the one
+// proposed in EIP-7745 for tree-hashing the filter map structure and for the
+// same data proximity reasons it is also suitable for database representation.
+// See also:
+// https://eips.ethereum.org/EIPS/eip-7745#hash-tree-structure
+func ReadFilterMapExtRow(db ethdb.KeyValueReader, mapRowIndex uint64, bitLength uint) ([]uint32, error) {
+ byteLength := int(bitLength) / 8
+ if int(bitLength) != byteLength*8 {
+ panic("invalid bit length")
+ }
+ key := filterMapRowKey(mapRowIndex, false)
+ has, err := db.Has(key)
+ if err != nil {
+ return nil, err
+ }
+ if !has {
+ return nil, nil
+ }
+ encRow, err := db.Get(key)
+ if err != nil {
+ return nil, err
+ }
+ if len(encRow)%byteLength != 0 {
+ return nil, errors.New("Invalid encoded extended filter row length")
+ }
+ row := make([]uint32, len(encRow)/byteLength)
+ var b [4]byte
+ for i := range row {
+ copy(b[:byteLength], encRow[i*byteLength:(i+1)*byteLength])
+ row[i] = binary.LittleEndian.Uint32(b[:])
+ }
+ return row, nil
+}
+
+func ReadFilterMapBaseRows(db ethdb.KeyValueReader, mapRowIndex uint64, rowCount uint32, bitLength uint) ([][]uint32, error) {
+ byteLength := int(bitLength) / 8
+ if int(bitLength) != byteLength*8 {
+ panic("invalid bit length")
+ }
+ key := filterMapRowKey(mapRowIndex, true)
+ has, err := db.Has(key)
+ if err != nil {
+ return nil, err
+ }
+ rows := make([][]uint32, rowCount)
+ if !has {
+ return rows, nil
+ }
+ encRows, err := db.Get(key)
+ if err != nil {
+ return nil, err
+ }
+ encLen := len(encRows)
+ var (
+ entryCount, entriesInRow, rowIndex, headerLen, headerBits int
+ headerByte byte
+ )
+ for headerLen+byteLength*entryCount < encLen {
+ if headerBits == 0 {
+ headerByte = encRows[headerLen]
+ headerLen++
+ headerBits = 8
+ }
+ if headerByte&1 > 0 {
+ entriesInRow++
+ entryCount++
+ } else {
+ if entriesInRow > 0 {
+ rows[rowIndex] = make([]uint32, entriesInRow)
+ entriesInRow = 0
+ }
+ rowIndex++
+ }
+ headerByte >>= 1
+ headerBits--
+ }
+ if headerLen+byteLength*entryCount > encLen {
+ return nil, errors.New("Invalid encoded base filter rows length")
+ }
+ if entriesInRow > 0 {
+ rows[rowIndex] = make([]uint32, entriesInRow)
+ }
+ nextEntry := headerLen
+ for _, row := range rows {
+ for i := range row {
+ var b [4]byte
+ copy(b[:byteLength], encRows[nextEntry:nextEntry+byteLength])
+ row[i] = binary.LittleEndian.Uint32(b[:])
+ nextEntry += byteLength
+ }
+ }
+ return rows, nil
+}
+
+// WriteFilterMapRow stores a filter map row at the given mapRowIndex or deletes
+// any existing entry if the row is empty.
+func WriteFilterMapExtRow(db ethdb.KeyValueWriter, mapRowIndex uint64, row []uint32, bitLength uint) {
+ byteLength := int(bitLength) / 8
+ if int(bitLength) != byteLength*8 {
+ panic("invalid bit length")
+ }
+ var err error
+ if len(row) > 0 {
+ encRow := make([]byte, len(row)*byteLength)
+ for i, c := range row {
+ var b [4]byte
+ binary.LittleEndian.PutUint32(b[:], c)
+ copy(encRow[i*byteLength:(i+1)*byteLength], b[:byteLength])
+ }
+ err = db.Put(filterMapRowKey(mapRowIndex, false), encRow)
+ } else {
+ err = db.Delete(filterMapRowKey(mapRowIndex, false))
+ }
+ if err != nil {
+ log.Crit("Failed to store extended filter map row", "err", err)
+ }
+}
+
+func WriteFilterMapBaseRows(db ethdb.KeyValueWriter, mapRowIndex uint64, rows [][]uint32, bitLength uint) {
+ byteLength := int(bitLength) / 8
+ if int(bitLength) != byteLength*8 {
+ panic("invalid bit length")
+ }
+ var entryCount, zeroBits int
+ for i, row := range rows {
+ if len(row) > 0 {
+ entryCount += len(row)
+ zeroBits = i
+ }
+ }
+ var err error
+ if entryCount > 0 {
+ headerLen := (zeroBits + entryCount + 7) / 8
+ encRows := make([]byte, headerLen+entryCount*byteLength)
+ nextEntry := headerLen
+
+ headerPtr, headerByte := 0, byte(1)
+ addHeaderBit := func(bit bool) {
+ if bit {
+ encRows[headerPtr] += headerByte
+ }
+ if headerByte += headerByte; headerByte == 0 {
+ headerPtr++
+ headerByte = 1
+ }
+ }
+
+ for _, row := range rows {
+ for _, entry := range row {
+ var b [4]byte
+ binary.LittleEndian.PutUint32(b[:], entry)
+ copy(encRows[nextEntry:nextEntry+byteLength], b[:byteLength])
+ nextEntry += byteLength
+ addHeaderBit(true)
+ }
+ if zeroBits == 0 {
+ break
+ }
+ addHeaderBit(false)
+ zeroBits--
+ }
+ err = db.Put(filterMapRowKey(mapRowIndex, true), encRows)
+ } else {
+ err = db.Delete(filterMapRowKey(mapRowIndex, true))
+ }
+ if err != nil {
+ log.Crit("Failed to store base filter map rows", "err", err)
+ }
+}
+
+func DeleteFilterMapRows(db ethdb.KeyValueRangeDeleter, firstMapRowIndex, afterLastMapRowIndex uint64) {
+ if err := db.DeleteRange(filterMapRowKey(firstMapRowIndex, false), filterMapRowKey(afterLastMapRowIndex, false)); err != nil {
+ log.Crit("Failed to delete range of filter map rows", "err", err)
+ }
+}
+
+// ReadFilterMapLastBlock retrieves the number of the block that generated the
+// last log value entry of the given map.
+func ReadFilterMapLastBlock(db ethdb.KeyValueReader, mapIndex uint32) (uint64, common.Hash, error) {
+ enc, err := db.Get(filterMapLastBlockKey(mapIndex))
+ if err != nil {
+ return 0, common.Hash{}, err
+ }
+ if len(enc) != 40 {
+ return 0, common.Hash{}, errors.New("Invalid block number and id encoding")
+ }
+ var id common.Hash
+ copy(id[:], enc[8:])
+ return binary.BigEndian.Uint64(enc[:8]), id, nil
+}
+
+// WriteFilterMapLastBlock stores the number of the block that generated the
+// last log value entry of the given map.
+func WriteFilterMapLastBlock(db ethdb.KeyValueWriter, mapIndex uint32, blockNumber uint64, id common.Hash) {
+ var enc [40]byte
+ binary.BigEndian.PutUint64(enc[:8], blockNumber)
+ copy(enc[8:], id[:])
+ if err := db.Put(filterMapLastBlockKey(mapIndex), enc[:]); err != nil {
+ log.Crit("Failed to store filter map last block pointer", "err", err)
+ }
+}
+
+// DeleteFilterMapLastBlock deletes the number of the block that generated the
+// last log value entry of the given map.
+func DeleteFilterMapLastBlock(db ethdb.KeyValueWriter, mapIndex uint32) {
+ if err := db.Delete(filterMapLastBlockKey(mapIndex)); err != nil {
+ log.Crit("Failed to delete filter map last block pointer", "err", err)
+ }
+}
+
+func DeleteFilterMapLastBlocks(db ethdb.KeyValueRangeDeleter, firstMapIndex, afterLastMapIndex uint32) {
+ if err := db.DeleteRange(filterMapLastBlockKey(firstMapIndex), filterMapLastBlockKey(afterLastMapIndex)); err != nil {
+ log.Crit("Failed to delete range of filter map last block pointers", "err", err)
+ }
+}
+
+// ReadBlockLvPointer retrieves the starting log value index where the log values
+// generated by the given block are located.
+func ReadBlockLvPointer(db ethdb.KeyValueReader, blockNumber uint64) (uint64, error) {
+ encPtr, err := db.Get(filterMapBlockLVKey(blockNumber))
+ if err != nil {
+ return 0, err
+ }
+ if len(encPtr) != 8 {
+ return 0, errors.New("Invalid log value pointer encoding")
+ }
+ return binary.BigEndian.Uint64(encPtr), nil
+}
+
+// WriteBlockLvPointer stores the starting log value index where the log values
+// generated by the given block are located.
+func WriteBlockLvPointer(db ethdb.KeyValueWriter, blockNumber, lvPointer uint64) {
+ var encPtr [8]byte
+ binary.BigEndian.PutUint64(encPtr[:], lvPointer)
+ if err := db.Put(filterMapBlockLVKey(blockNumber), encPtr[:]); err != nil {
+ log.Crit("Failed to store block log value pointer", "err", err)
+ }
+}
+
+// DeleteBlockLvPointer deletes the starting log value index where the log values
+// generated by the given block are located.
+func DeleteBlockLvPointer(db ethdb.KeyValueWriter, blockNumber uint64) {
+ if err := db.Delete(filterMapBlockLVKey(blockNumber)); err != nil {
+ log.Crit("Failed to delete block log value pointer", "err", err)
+ }
+}
+
+func DeleteBlockLvPointers(db ethdb.KeyValueRangeDeleter, firstBlockNumber, afterLastBlockNumber uint64) {
+ if err := db.DeleteRange(filterMapBlockLVKey(firstBlockNumber), filterMapBlockLVKey(afterLastBlockNumber)); err != nil {
+ log.Crit("Failed to delete range of block log value pointers", "err", err)
+ }
+}
+
+// FilterMapsRange is a storage representation of the block range covered by the
+// filter maps structure and the corresponting log value index range.
+type FilterMapsRange struct {
+ HeadBlockIndexed bool
+ HeadBlockDelimiter uint64
+ FirstIndexedBlock, AfterLastIndexedBlock uint64
+ FirstRenderedMap, AfterLastRenderedMap, TailPartialEpoch uint32
+}
+
+// ReadFilterMapsRange retrieves the filter maps range data. Note that if the
+// database entry is not present, that is interpreted as a valid non-initialized
+// state and returns a blank range structure and no error.
+func ReadFilterMapsRange(db ethdb.KeyValueReader) (FilterMapsRange, bool, error) {
+ if has, err := db.Has(filterMapsRangeKey); !has || err != nil {
+ return FilterMapsRange{}, false, err
+ }
+ encRange, err := db.Get(filterMapsRangeKey)
+ if err != nil {
+ return FilterMapsRange{}, false, err
+ }
+ var fmRange FilterMapsRange
+ if err := rlp.DecodeBytes(encRange, &fmRange); err != nil {
+ return FilterMapsRange{}, false, err
+ }
+ return fmRange, true, err
+}
+
+// WriteFilterMapsRange stores the filter maps range data.
+func WriteFilterMapsRange(db ethdb.KeyValueWriter, fmRange FilterMapsRange) {
+ encRange, err := rlp.EncodeToBytes(&fmRange)
+ if err != nil {
+ log.Crit("Failed to encode filter maps range", "err", err)
+ }
+ if err := db.Put(filterMapsRangeKey, encRange); err != nil {
+ log.Crit("Failed to store filter maps range", "err", err)
+ }
+}
+
+// DeleteFilterMapsRange deletes the filter maps range data which is interpreted
+// as reverting to the un-initialized state.
+func DeleteFilterMapsRange(db ethdb.KeyValueWriter) {
+ if err := db.Delete(filterMapsRangeKey); err != nil {
+ log.Crit("Failed to delete filter maps range", "err", err)
+ }
+}
diff --git a/core/rawdb/database.go b/core/rawdb/database.go
index ae1e7c0bdc9..bc293473248 100644
--- a/core/rawdb/database.go
+++ b/core/rawdb/database.go
@@ -376,6 +376,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
storageSnaps stat
preimages stat
bloomBits stat
+ filterMaps stat
beaconHeaders stat
cliqueSnaps stat
@@ -440,6 +441,8 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
bloomBits.Add(size)
case bytes.HasPrefix(key, BloomBitsIndexPrefix):
bloomBits.Add(size)
+ case bytes.HasPrefix(key, []byte(FilterMapsPrefix)):
+ filterMaps.Add(size)
case bytes.HasPrefix(key, skeletonHeaderPrefix) && len(key) == (len(skeletonHeaderPrefix)+8):
beaconHeaders.Add(size)
case bytes.HasPrefix(key, CliqueSnapshotPrefix) && len(key) == 7+common.HashLength:
@@ -505,6 +508,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
{"Key-Value store", "Block hash->number", hashNumPairings.Size(), hashNumPairings.Count()},
{"Key-Value store", "Transaction index", txLookups.Size(), txLookups.Count()},
{"Key-Value store", "Bloombit index", bloomBits.Size(), bloomBits.Count()},
+ {"Key-Value store", "Log search index", filterMaps.Size(), filterMaps.Count()},
{"Key-Value store", "Contract codes", codes.Size(), codes.Count()},
{"Key-Value store", "Hash trie nodes", legacyTries.Size(), legacyTries.Count()},
{"Key-Value store", "Path trie state lookups", stateLookups.Size(), stateLookups.Count()},
diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go
index 5480c7a72c6..b41951113ce 100644
--- a/core/rawdb/schema.go
+++ b/core/rawdb/schema.go
@@ -145,6 +145,12 @@ var (
FixedCommitteeRootKey = []byte("fixedRoot-") // bigEndian64(syncPeriod) -> committee root hash
SyncCommitteeKey = []byte("committee-") // bigEndian64(syncPeriod) -> serialized committee
+ FilterMapsPrefix = "fm-"
+ filterMapsRangeKey = []byte(FilterMapsPrefix + "R")
+ filterMapRowPrefix = []byte(FilterMapsPrefix + "r") // filterMapRowPrefix + mapRowIndex (uint64 big endian) -> filter row
+ filterMapLastBlockPrefix = []byte(FilterMapsPrefix + "b") // filterMapLastBlockPrefix + mapIndex (uint32 big endian) -> block number (uint64 big endian)
+ filterMapBlockLVPrefix = []byte(FilterMapsPrefix + "p") // filterMapBlockLVPrefix + num (uint64 big endian) -> log value pointer (uint64 big endian)
+
preimageCounter = metrics.NewRegisteredCounter("db/preimage/total", nil)
preimageHitsCounter = metrics.NewRegisteredCounter("db/preimage/hits", nil)
preimageMissCounter = metrics.NewRegisteredCounter("db/preimage/miss", nil)
@@ -342,3 +348,34 @@ func IsStorageTrieNode(key []byte) bool {
ok, _, _ := ResolveStorageTrieNode(key)
return ok
}
+
+// filterMapRowKey = filterMapRowPrefix + mapRowIndex (uint64 big endian)
+func filterMapRowKey(mapRowIndex uint64, base bool) []byte {
+ extLen := 8
+ if base {
+ extLen = 9
+ }
+ l := len(filterMapRowPrefix)
+ key := make([]byte, l+extLen)
+ copy(key[:l], filterMapRowPrefix)
+ binary.BigEndian.PutUint64(key[l:l+8], mapRowIndex)
+ return key
+}
+
+// filterMapLastBlockKey = filterMapLastBlockPrefix + mapIndex (uint32 big endian)
+func filterMapLastBlockKey(mapIndex uint32) []byte {
+ l := len(filterMapLastBlockPrefix)
+ key := make([]byte, l+4)
+ copy(key[:l], filterMapLastBlockPrefix)
+ binary.BigEndian.PutUint32(key[l:], mapIndex)
+ return key
+}
+
+// filterMapBlockLVKey = filterMapBlockLVPrefix + num (uint64 big endian)
+func filterMapBlockLVKey(number uint64) []byte {
+ l := len(filterMapBlockLVPrefix)
+ key := make([]byte, l+8)
+ copy(key[:l], filterMapBlockLVPrefix)
+ binary.BigEndian.PutUint64(key[l:], number)
+ return key
+}