update to latest light client libp2p protocol

Incorporates the latest changes to the light client sync protocol based
on Devconnect AMS feedback. Note that this breaks compatibility with the
previous prototype, due to changes to data structures and endpoints.
See ethereum/consensus-specs#2802

Due to the amount of changes, it is best to review the heavily edited
files from scratch (likewise, for the spec).

Author: etan-status

AllTests-mainnet.md

@@ -257,11 +257,11 @@ OK: 3/3 Fail: 0/3 Skip: 0/3
 ## Light client processor [Preset: mainnet]
 ```diff
 + Duplicate bootstrap [Preset: mainnet]                                                      OK
-+ Forced update [Preset: mainnet]                                                            OK
 + Invalid bootstrap [Preset: mainnet]                                                        OK
++ Missing bootstrap (finality update) [Preset: mainnet]                                      OK
 + Missing bootstrap (optimistic update) [Preset: mainnet]                                    OK
 + Missing bootstrap (update) [Preset: mainnet]                                               OK
-+ Standard sync [Preset: mainnet]                                                            OK
++ Sync [Preset: mainnet]                                                                     OK
 ```
 OK: 6/6 Fail: 0/6 Skip: 0/6
 ## ListKeys requests [Preset: mainnet]

ConsensusSpecPreset-mainnet.md

@@ -804,6 +804,11 @@ OK: 35/35 Fail: 0/35 Skip: 0/35
   All tests                                                                                  Skip
 ```
 OK: 0/1 Fail: 0/1 Skip: 1/1
+## EF - Altair - Sync protocol - Update ranking [Preset: mainnet]
+```diff
+  All tests                                                                                  Skip
+```
+OK: 0/1 Fail: 0/1 Skip: 1/1
 ## EF - Altair - Unittests - Sync protocol [Preset: mainnet]
 ```diff
 + process_light_client_update_finality_updated                                               OK
@@ -1215,4 +1220,4 @@ OK: 44/44 Fail: 0/44 Skip: 0/44
 OK: 27/27 Fail: 0/27 Skip: 0/27
 
 ---TOTAL---
-OK: 1035/1037 Fail: 0/1037 Skip: 2/1037
+OK: 1035/1038 Fail: 0/1038 Skip: 3/1038

ConsensusSpecPreset-minimal.md

@@ -845,6 +845,11 @@ OK: 35/35 Fail: 0/35 Skip: 0/35
   All tests                                                                                  Skip
 ```
 OK: 0/1 Fail: 0/1 Skip: 1/1
+## EF - Altair - Sync protocol - Update ranking [Preset: minimal]
+```diff
+  All tests                                                                                  Skip
+```
+OK: 0/1 Fail: 0/1 Skip: 1/1
 ## EF - Altair - Unittests - Sync protocol [Preset: minimal]
 ```diff
 + process_light_client_update_finality_updated                                               OK
@@ -1292,4 +1297,4 @@ OK: 48/48 Fail: 0/48 Skip: 0/48
 OK: 30/30 Fail: 0/30 Skip: 0/30
 
 ---TOTAL---
-OK: 1085/1106 Fail: 0/1106 Skip: 21/1106
+OK: 1085/1107 Fail: 0/1107 Skip: 22/1107

beacon_chain/consensus_object_pools/block_pools_types.nim

@@ -216,8 +216,10 @@ type
       ## On beacon chain reorganization
     onFinHappened*: OnFinalizedCallback
       ## On finalization callback
-    onOptimisticLightClientUpdate*: OnOptimisticLightClientUpdateCallback
-      ## On `OptimisticLightClientUpdate` updated callback
+    onLightClientFinalityUpdate*: OnLightClientFinalityUpdateCallback
+      ## On new `LightClientFinalityUpdate` callback
+    onLightClientOptimisticUpdate*: OnLightClientOptimisticUpdateCallback
+      ## On new `LightClientOptimisticUpdate` callback
 
     headSyncCommittees*: SyncCommitteeCache
       ## A cache of the sync committees, as they appear in the head state -

beacon_chain/consensus_object_pools/block_pools_types_light_client.nim

@@ -13,13 +13,16 @@
 import
   # Status libraries
   stew/bitops2,
+  chronos,
   # Beacon chain internals
   ../spec/datatypes/altair,
   ./block_dag
 
 type
-  OnOptimisticLightClientUpdateCallback* =
-    proc(data: OptimisticLightClientUpdate) {.gcsafe, raises: [Defect].}
+  OnLightClientFinalityUpdateCallback* =
+    proc(data: altair.LightClientFinalityUpdate) {.gcsafe, raises: [Defect].}
+  OnLightClientOptimisticUpdateCallback* =
+    proc(data: altair.LightClientOptimisticUpdate) {.gcsafe, raises: [Defect].}
 
   ImportLightClientData* {.pure.} = enum
     ## Controls which classes of light client data are imported.
@@ -30,18 +33,17 @@ type
     Full = "full"
       ## Import light client data for entire weak subjectivity period.
     OnDemand = "on-demand"
-      ## No precompute of historic data. Is slow and may miss validator duties.
+      ## Don't precompute historic data. Slow, may miss validator duties.
 
   CachedLightClientData* = object
     ## Cached data from historical non-finalized states to improve speed when
     ## creating future `LightClientUpdate` and `LightClientBootstrap` instances.
     current_sync_committee_branch*:
       array[log2trunc(altair.CURRENT_SYNC_COMMITTEE_INDEX), Eth2Digest]
-
     next_sync_committee_branch*:
       array[log2trunc(altair.NEXT_SYNC_COMMITTEE_INDEX), Eth2Digest]
 
-    finalized_bid*: BlockId
+    finalized_slot*: Slot
     finality_branch*:
       array[log2trunc(altair.FINALIZED_ROOT_INDEX), Eth2Digest]
 
@@ -55,40 +57,44 @@ type
     data*: Table[BlockId, CachedLightClientData]
       ## Cached data for creating future `LightClientUpdate` instances.
       ## Key is the block ID of which the post state was used to get the data.
-      ## Data is stored for the most recent 4 finalized checkpoints, as well as
-      ## for all non-finalized blocks.
+      ## Data stored for the finalized head block and all non-finalized blocks.
 
     bootstrap*: Table[Slot, CachedLightClientBootstrap]
       ## Cached data for creating future `LightClientBootstrap` instances.
       ## Key is the block slot of which the post state was used to get the data.
-      ## Data is stored for finalized epoch boundary blocks.
-
-    latestCheckpoints*: array[4, Checkpoint]
-      ## Keeps track of the latest four `finalized_checkpoint` references
-      ## leading to `finalizedHead`. Used to prune `data`.
-      ## Non-finalized states may only refer to these checkpoints.
+      ## Data stored for all finalized epoch boundary blocks.
 
-    lastCheckpointIndex*: int
-      ## Last index that was modified in `latestCheckpoints`.
-
-    bestUpdates*: Table[SyncCommitteePeriod, altair.LightClientUpdate]
+    best*: Table[SyncCommitteePeriod, altair.LightClientUpdate]
       ## Stores the `LightClientUpdate` with the most `sync_committee_bits` per
-      ## `SyncCommitteePeriod`. Updates with a finality proof have precedence.
+      ## `SyncCommitteePeriod`. Sync committee finality gives precedence.
 
-    pendingBestUpdates*:
+    pendingBest*:
       Table[(SyncCommitteePeriod, Eth2Digest), altair.LightClientUpdate]
-      ## Same as `bestUpdates`, but for `SyncCommitteePeriod` with
-      ## `next_sync_committee` that are not finalized. Key is `(period,
+      ## Same as `best`, but for `SyncCommitteePeriod` with not yet finalized
+      ## `next_sync_committee`. Key is `(attested_period,
       ## hash_tree_root(current_sync_committee | next_sync_committee)`.
 
-    latestUpdate*: altair.LightClientUpdate
-      ## Tracks the `LightClientUpdate` for the latest slot. This may be older
-      ## than head for empty slots or if not signed by sync committee.
-
-    optimisticUpdate*: OptimisticLightClientUpdate
-      ## Tracks the `OptimisticLightClientUpdate` for the latest slot. This may
-      ## be older than head for empty slots or if not signed by sync committee.
+    latest*: altair.LightClientFinalityUpdate
+      ## Tracks light client data for the latest slot that was signed by
+      ## at least `MIN_SYNC_COMMITTEE_PARTICIPANTS`. May be older than head.
 
     importTailSlot*: Slot
-      ## The earliest slot for which light client data is collected.
-      ## Only relevant for `ImportLightClientData.OnlyNew`.
+      ## The earliest slot for which light client data is imported.
+
+    latestForwardedFinalitySlot*: Slot
+      ## Latest finality update that was forwarded on libp2p gossip.
+      ## Tracks `finality_update.finalized_header.slot`.
+
+    latestForwardedOptimisticSlot*: Slot
+      ## Latest optimistic update that was forwarded on libp2p gossip.
+      ## Tracks `optimistic_update.attested_header.slot`.
+
+    latestBroadcastedSlot*: Slot
+      ## Latest slot for which updates were broadcasted on libp2p gossip.
+      ## Tracks `update.signature_slot`.
+
+    broadcastGossipFut*: Future[void]
+      ## Task to broadcast libp2p gossip. Started when a sync committee message
+      ## is sent. Tracked separately from `handleValidatorDuties` to catch the
+      ## case where `node.attachedValidators[].count == 0` at function start,
+      ## and then a sync committee message gets sent from a remote VC via REST.

beacon_chain/consensus_object_pools/blockchain_dag.nim

@@ -556,10 +556,10 @@ proc updateBeaconMetrics(
 import blockchain_dag_light_client
 
 export
-  blockchain_dag_light_client.getBestLightClientUpdateForPeriod,
-  blockchain_dag_light_client.getLatestLightClientUpdate,
-  blockchain_dag_light_client.getOptimisticLightClientUpdate,
-  blockchain_dag_light_client.getLightClientBootstrap
+  blockchain_dag_light_client.getLightClientBootstrap,
+  blockchain_dag_light_client.getLightClientUpdateForPeriod,
+  blockchain_dag_light_client.getLightClientFinalityUpdate,
+  blockchain_dag_light_client.getLightClientOptimisticUpdate
 
 proc getViableHead(cfg: RuntimeConfig, db: BeaconChainDB): Opt[BlockId] =
   # When the database has been written with a pre-fork version of the
@@ -659,9 +659,9 @@ proc init*(T: type ChainDAGRef, cfg: RuntimeConfig, db: BeaconChainDB,
            validatorMonitor: ref ValidatorMonitor, updateFlags: UpdateFlags,
            eraPath = ".",
            onBlockCb: OnBlockCallback = nil, onHeadCb: OnHeadCallback = nil,
-           onReorgCb: OnReorgCallback = nil,
-           onFinCb: OnFinalizedCallback = nil,
-           onOptimisticLCUpdateCb: OnOptimisticLightClientUpdateCallback = nil,
+           onReorgCb: OnReorgCallback = nil, onFinCb: OnFinalizedCallback = nil,
+           onLCFinalityUpdateCb: OnLightClientFinalityUpdateCallback = nil,
+           onLCOptimisticUpdateCb: OnLightClientOptimisticUpdateCallback = nil,
            serveLightClientData = false,
            importLightClientData = ImportLightClientData.None): ChainDAGRef =
   # TODO move fork version sanity checking elsewhere?
@@ -712,7 +712,8 @@ proc init*(T: type ChainDAGRef, cfg: RuntimeConfig, db: BeaconChainDB,
       onHeadChanged: onHeadCb,
       onReorgHappened: onReorgCb,
       onFinHappened: onFinCb,
-      onOptimisticLightClientUpdate: onOptimisticLCUpdateCb
+      onLightClientFinalityUpdate: onLCFinalityUpdateCb,
+      onLightClientOptimisticUpdate: onLCOptimisticUpdateCb
     )
     loadTick = Moment.now()
 
@@ -1670,18 +1671,18 @@ proc updateHead*(
         tmp.parent = nil # Reset all parent links to release memory
         tmp = p
 
-      dag.finalizedHead = finalizedHead
-
       dag.db.updateFinalizedBlocks(newFinalized)
 
+    # Update light client data
+    dag.processFinalizationForLightClient(finalizedHead)
+
+    dag.finalizedHead = finalizedHead
+
     # Pruning the block dag is required every time the finalized head changes
     # in order to clear out blocks that are no longer viable and should
     # therefore no longer be considered as part of the chain we're following
     dag.pruneBlocksDAG()
 
-    # Update light client data
-    dag.processFinalizationForLightClient()
-
     # Send notification about new finalization point via callback.
     if not(isNil(dag.onFinHappened)):
       let stateRoot =