fix: refact rln-relay and post sync test (#3434)

Author: darshankabariya

apps/benchmarks/benchmarks.nim

@@ -1,49 +1,66 @@
 import
-  math,
-  std/sequtils,
-  results,
-  options,
+  std/[strutils, times, sequtils, osproc], math, results, options, testutils/unittests
+
+import
   waku/[
     waku_rln_relay/protocol_types,
     waku_rln_relay/rln,
     waku_rln_relay,
     waku_rln_relay/conversion_utils,
-    waku_rln_relay/group_manager/static/group_manager,
-  ]
-
-import std/[times, os]
+    waku_rln_relay/group_manager/on_chain/group_manager,
+  ],
+  tests/waku_rln_relay/utils_onchain
 
 proc main(): Future[string] {.async, gcsafe.} =
-  let rlnIns = createRLNInstance(20).get()
-  let credentials = toSeq(0 .. 1000).mapIt(membershipKeyGen(rlnIns).get())
+  # Spin up a local Ethereum JSON-RPC (Anvil) and deploy the RLN contract
+  let anvilProc = runAnvil()
+  defer:
+    stopAnvil(anvilProc)
+
+  # Set up an On-chain group manager (includes contract deployment)
+  let manager = await setupOnchainGroupManager()
+  (await manager.init()).isOkOr:
+    raiseAssert $error
 
-  let manager = StaticGroupManager(
-    rlnInstance: rlnIns,
-    groupSize: 1000,
-    membershipIndex: some(MembershipIndex(900)),
-    groupKeys: credentials,
-  )
+  # Register a new member so that we can later generate proofs
+  let idCredentials = generateCredentials(manager.rlnInstance)
 
-  await manager.init()
+  try:
+    await manager.register(idCredentials, UserMessageLimit(100))
+  except Exception, CatchableError:
+    assert false, "exception raised: " & getCurrentExceptionMsg()
+
+  let rootUpdated = await manager.updateRoots()
 
+  if rootUpdated:
+    let proofResult = await manager.fetchMerkleProofElements()
+    if proofResult.isErr():
+      error "Failed to fetch Merkle proof", error = proofResult.error
+    manager.merkleProofCache = proofResult.get()
+
+  let epoch = default(Epoch)
+  debug "epoch in bytes", epochHex = epoch.inHex()
   let data: seq[byte] = newSeq[byte](1024)
 
   var proofGenTimes: seq[times.Duration] = @[]
   var proofVerTimes: seq[times.Duration] = @[]
-  for i in 0 .. 50:
+
+  for i in 1 .. 100:
     var time = getTime()
-    let proof = manager.generateProof(data, default(Epoch)).get()
+    let proof = manager.generateProof(data, epoch, MessageId(i.uint8)).valueOr:
+      raiseAssert $error
     proofGenTimes.add(getTime() - time)
 
     time = getTime()
-    let res = manager.verifyProof(data, proof).get()
+    let ok = manager.verifyProof(data, proof).valueOr:
+      raiseAssert $error
     proofVerTimes.add(getTime() - time)
 
   echo "Proof generation times: ", sum(proofGenTimes) div len(proofGenTimes)
   echo "Proof verification times: ", sum(proofVerTimes) div len(proofVerTimes)
 
 when isMainModule:
   try:
-    waitFor(main())
+    discard waitFor main()
   except CatchableError as e:
     raise e

tests/node/test_wakunode_legacy_lightpush.nim

@@ -1,7 +1,7 @@
 {.used.}
 
 import
-  std/[options, tempfiles, net],
+  std/[options, tempfiles, net, osproc],
   testutils/unittests,
   chronos,
   std/strformat,
@@ -17,8 +17,9 @@ import
     waku_lightpush_legacy/protocol_metrics,
     waku_rln_relay,
   ],
-  ../testlib/[wakucore, wakunode, testasync, futures],
-  ../resources/payloads
+  ../testlib/[wakucore, wakunode, testasync, futures, testutils],
+  ../resources/payloads,
+  ../waku_rln_relay/[rln/waku_rln_relay_utils, utils_onchain]
 
 suite "Waku Legacy Lightpush - End To End":
   var
@@ -110,6 +111,8 @@ suite "RLN Proofs as a Lightpush Service":
 
     server {.threadvar.}: WakuNode
     client {.threadvar.}: WakuNode
+    anvilProc {.threadvar.}: Process
+    manager {.threadvar.}: OnchainGroupManager
 
     serverRemotePeerInfo {.threadvar.}: RemotePeerInfo
     pubsubTopic {.threadvar.}: PubsubTopic
@@ -131,13 +134,14 @@ suite "RLN Proofs as a Lightpush Service":
     server = newTestWakuNode(serverKey, parseIpAddress("0.0.0.0"), Port(0))
     client = newTestWakuNode(clientKey, parseIpAddress("0.0.0.0"), Port(0))
 
+    anvilProc = runAnvil()
+    manager = waitFor setupOnchainGroupManager()
+
     # mount rln-relay
-    let wakuRlnConfig = WakuRlnConfig(
-      dynamic: false,
-      credIndex: some(1.uint),
-      userMessageLimit: 1,
-      epochSizeSec: 1,
-      treePath: genTempPath("rln_tree", "wakunode"),
+    let wakuRlnConfig = getWakuRlnConfig(
+      manager = manager,
+      treePath = genTempPath("rln_tree", "wakunode_1"),
+      index = MembershipIndex(1),
     )
 
     await allFutures(server.start(), client.start())
@@ -149,13 +153,32 @@ suite "RLN Proofs as a Lightpush Service":
     await server.mountLegacyLightPush()
     client.mountLegacyLightPushClient()
 
+    let manager1 = cast[OnchainGroupManager](server.wakuRlnRelay.groupManager)
+    let idCredentials1 = generateCredentials(manager1.rlnInstance)
+
+    try:
+      waitFor manager1.register(idCredentials1, UserMessageLimit(20))
+    except Exception, CatchableError:
+      assert false,
+        "exception raised when calling register: " & getCurrentExceptionMsg()
+
+    let rootUpdated1 = waitFor manager1.updateRoots()
+    debug "Updated root for node1", rootUpdated1
+
+    if rootUpdated1:
+      let proofResult = waitFor manager1.fetchMerkleProofElements()
+      if proofResult.isErr():
+        error "Failed to fetch Merkle proof", error = proofResult.error
+      manager1.merkleProofCache = proofResult.get()
+
     serverRemotePeerInfo = server.peerInfo.toRemotePeerInfo()
     pubsubTopic = DefaultPubsubTopic
     contentTopic = DefaultContentTopic
     message = fakeWakuMessage()
 
   asyncTeardown:
     await server.stop()
+    stopAnvil(anvilProc)
 
   suite "Lightpush attaching RLN proofs":
     asyncTest "Message is published when RLN enabled":

tests/node/test_wakunode_lightpush.nim

@@ -1,7 +1,7 @@
 {.used.}
 
 import
-  std/[options, tempfiles],
+  std/[options, tempfiles, osproc],
   testutils/unittests,
   chronos,
   std/strformat,
@@ -10,7 +10,8 @@ import
 import
   waku/[waku_core, node/peer_manager, node/waku_node, waku_lightpush, waku_rln_relay],
   ../testlib/[wakucore, wakunode, testasync, futures],
-  ../resources/payloads
+  ../resources/payloads,
+  ../waku_rln_relay/[rln/waku_rln_relay_utils, utils_onchain]
 
 const PublishedToOnePeer = 1
 
@@ -104,6 +105,8 @@ suite "RLN Proofs as a Lightpush Service":
 
     server {.threadvar.}: WakuNode
     client {.threadvar.}: WakuNode
+    anvilProc {.threadvar.}: Process
+    manager {.threadvar.}: OnchainGroupManager
 
     serverRemotePeerInfo {.threadvar.}: RemotePeerInfo
     pubsubTopic {.threadvar.}: PubsubTopic
@@ -125,13 +128,14 @@ suite "RLN Proofs as a Lightpush Service":
     server = newTestWakuNode(serverKey, parseIpAddress("0.0.0.0"), Port(0))
     client = newTestWakuNode(clientKey, parseIpAddress("0.0.0.0"), Port(0))
 
+    anvilProc = runAnvil()
+    manager = waitFor setupOnchainGroupManager()
+
     # mount rln-relay
-    let wakuRlnConfig = WakuRlnConfig(
-      dynamic: false,
-      credIndex: some(1.uint),
-      userMessageLimit: 1,
-      epochSizeSec: 1,
-      treePath: genTempPath("rln_tree", "wakunode"),
+    let wakuRlnConfig = getWakuRlnConfig(
+      manager = manager,
+      treePath = genTempPath("rln_tree", "wakunode_1"),
+      index = MembershipIndex(1),
     )
 
     await allFutures(server.start(), client.start())
@@ -143,13 +147,32 @@ suite "RLN Proofs as a Lightpush Service":
     await server.mountLightPush()
     client.mountLightPushClient()
 
+    let manager1 = cast[OnchainGroupManager](server.wakuRlnRelay.groupManager)
+    let idCredentials1 = generateCredentials(manager1.rlnInstance)
+
+    try:
+      waitFor manager1.register(idCredentials1, UserMessageLimit(20))
+    except Exception, CatchableError:
+      assert false,
+        "exception raised when calling register: " & getCurrentExceptionMsg()
+
+    let rootUpdated1 = waitFor manager1.updateRoots()
+    debug "Updated root for node1", rootUpdated1
+
+    if rootUpdated1:
+      let proofResult = waitFor manager1.fetchMerkleProofElements()
+      if proofResult.isErr():
+        error "Failed to fetch Merkle proof", error = proofResult.error
+      manager1.merkleProofCache = proofResult.get()
+
     serverRemotePeerInfo = server.peerInfo.toRemotePeerInfo()
     pubsubTopic = DefaultPubsubTopic
     contentTopic = DefaultContentTopic
     message = fakeWakuMessage()
 
   asyncTeardown:
     await server.stop()
+    stopAnvil(anvilProc)
 
   suite "Lightpush attaching RLN proofs":
     asyncTest "Message is published when RLN enabled":

tests/waku_rln_relay/rln/test_wrappers.nim

@@ -15,7 +15,8 @@ import
   waku/waku_rln_relay/rln/wrappers,
   ./waku_rln_relay_utils,
   ../../testlib/[simple_mock, assertions],
-  ../../waku_keystore/utils
+  ../../waku_keystore/utils,
+  ../../testlib/testutils
 
 from std/times import epochTime
 
@@ -98,7 +99,7 @@ suite "RlnConfig":
   suite "createRLNInstance":
     test "ok":
       # When we create the RLN instance
-      let rlnRes: RLNResult = createRLNInstance(15, "my.db")
+      let rlnRes: RLNResult = createRLNInstance(15)
 
       # Then it succeeds
       check:
@@ -124,7 +125,7 @@ suite "RlnConfig":
         newCircuitMock
 
       # When we create the RLN instance
-      let rlnRes: RLNResult = createRLNInstance(15, "my.db")
+      let rlnRes: RLNResult = createRLNInstance(15)
 
       # Then it fails
       check:
@@ -133,42 +134,3 @@ suite "RlnConfig":
       # Cleanup
       mock(new_circuit):
         backup
-
-  suite "proofGen":
-    test "Valid zk proof":
-      # this test vector is from zerokit
-      let rlnInstanceRes = createRLNInstanceWrapper()
-      assertResultOk(rlnInstanceRes)
-      let rlnInstance = rlnInstanceRes.value
-
-      let identityCredential = defaultIdentityCredential()
-      assert rlnInstance.insertMember(identityCredential.idCommitment)
-
-      let merkleRootRes = rlnInstance.getMerkleRoot()
-      assertResultOk(merkleRootRes)
-      let merkleRoot = merkleRootRes.value
-
-      let proofGenRes = rlnInstance.proofGen(
-        data = @[],
-        memKeys = identityCredential,
-        memIndex = MembershipIndex(0),
-        epoch = uint64(epochTime() / 1.float64).toEpoch(),
-      )
-      assertResultOk(proofGenRes)
-
-      let
-        rateLimitProof = proofGenRes.value
-        proofVerifyRes = rlnInstance.proofVerify(
-          data = @[], proof = rateLimitProof, validRoots = @[merkleRoot]
-        )
-
-      assertResultOk(proofVerifyRes)
-      assert proofVerifyRes.value, "proof verification failed"
-
-      # Assert the proof fields adhere to the specified types and lengths
-      check:
-        typeEq(rateLimitProof.proof, array[256, byte])
-        typeEq(rateLimitProof.merkleRoot, array[32, byte])
-        typeEq(rateLimitProof.shareX, array[32, byte])
-        typeEq(rateLimitProof.shareY, array[32, byte])
-        typeEq(rateLimitProof.nullifier, array[32, byte])

tests/waku_rln_relay/rln/waku_rln_relay_utils.nim

@@ -1,23 +1,59 @@
 import std/tempfiles
 
-import waku/waku_rln_relay, waku/waku_rln_relay/[rln, protocol_types]
+import
+  waku/waku_rln_relay,
+  waku/waku_rln_relay/[
+    group_manager, rln, conversion_utils, constants, protocol_types, protocol_metrics,
+    nonce_manager,
+  ]
 
 proc createRLNInstanceWrapper*(): RLNResult =
   return createRlnInstance(tree_path = genTempPath("rln_tree", "waku_rln_relay"))
 
 proc unsafeAppendRLNProof*(
-    rlnPeer: WakuRLNRelay, msg: var WakuMessage, senderEpochTime: float64
+    rlnPeer: WakuRLNRelay, msg: var WakuMessage, epoch: Epoch, messageId: MessageId
 ): RlnRelayResult[void] =
-  ## this proc derived from appendRLNProof, does not perform nonce check to
-  ## facilitate bad message id generation for testing
+  ## Test helper derived from `appendRLNProof`.
+  ## - Skips nonce validation to intentionally allow generating "bad" message IDs for tests.
+  ## - Forces a real-time on-chain Merkle root refresh via `updateRoots()` and fetches Merkle
+  ##   proof elements, updating `merkleProofCache` (bypasses `trackRootsChanges`).
+  ## WARNING: For testing only
 
-  let input = msg.toRLNSignal()
-  let epoch = rlnPeer.calcEpoch(senderEpochTime)
+  let manager = cast[OnchainGroupManager](rlnPeer.groupManager)
+  let rootUpdated = waitFor manager.updateRoots()
 
-  # we do not fetch a nonce from the nonce manager,
-  # instead we use 0 as the nonce
-  let proof = rlnPeer.groupManager.generateProof(input, epoch, 0).valueOr:
+  # Fetch Merkle proof either when a new root was detected *or* when the cache is empty.
+  if rootUpdated or manager.merkleProofCache.len == 0:
+    let proofResult = waitFor manager.fetchMerkleProofElements()
+    if proofResult.isErr():
+      error "Failed to fetch Merkle proof", error = proofResult.error
+    manager.merkleProofCache = proofResult.get()
+
+  let proof = manager.generateProof(msg.toRLNSignal(), epoch, messageId).valueOr:
     return err("could not generate rln-v2 proof: " & $error)
 
   msg.proof = proof.encode().buffer
   return ok()
+
+proc getWakuRlnConfig*(
+    manager: OnchainGroupManager,
+    userMessageLimit: uint64 = 1,
+    epochSizeSec: uint64 = 1,
+    treePath: string = genTempPath("rln_tree", "waku_rln_relay"),
+    index: MembershipIndex = MembershipIndex(0),
+): WakuRlnConfig =
+  let wakuRlnConfig = WakuRlnConfig(
+    dynamic: true,
+    ethClientUrls: @[EthClient],
+    ethContractAddress: manager.ethContractAddress,
+    chainId: manager.chainId,
+    credIndex: some(index),
+    userMessageLimit: userMessageLimit,
+    epochSizeSec: epochSizeSec,
+    treePath: treePath,
+    ethPrivateKey: some(manager.ethPrivateKey.get()),
+    onFatalErrorAction: proc(errStr: string) =
+      warn "non-fatal onchain test error", errStr
+    ,
+  )
+  return wakuRlnConfig