chore: run with mysql

Author: vasco-santos

.aegir.js

@@ -8,7 +8,14 @@ const WebSockets = require('libp2p-websockets')
 const Muxer = require('libp2p-mplex')
 const { NOISE: Crypto } = require('libp2p-noise')
 
+const { isNode } = require('ipfs-utils/src/env')
+const delay = require('delay')
+const execa = require('execa')
+const pWaitFor = require('p-wait-for')
+const isCI = require('is-ci')
+
 let libp2p
+let containerId
 
 const before = async () => {
   // Use the last peer
@@ -36,10 +43,40 @@ const before = async () => {
   })
 
   await libp2p.start()
+
+  // CI runs datastore service
+  if (isCI || !isNode) {
+    return
+  }
+
+  const procResult = execa.commandSync('docker run -p 3306:3306 -e MYSQL_ROOT_PASSWORD=test-secret-pw -e MYSQL_DATABASE=libp2p_rendezvous_db -d mysql:8 --default-authentication-plugin=mysql_native_password', {
+    all: true
+  })
+  containerId = procResult.stdout
+
+  console.log(`wait for docker container ${containerId} to be ready`)
+
+  await pWaitFor(() => {
+    const procCheck = execa.commandSync(`docker logs ${containerId}`)
+    const logs = procCheck.stdout + procCheck.stderr // Docker/MySQL sends to the stderr the ready for connections...
+
+    return logs.includes('ready for connections')
+  }, {
+    interval: 5000
+  })
+  // Some more time waiting
+  await delay(10e3)
 }
 
 const after = async () => {
   await libp2p.stop()
+
+  if (isCI || !isNode) {
+    return
+  }
+
+  console.log('docker container is stopping')
+  execa.commandSync(`docker stop ${containerId}`)
 }
 
 module.exports = {

.github/workflows/main.yml

@@ -0,0 +1,62 @@
+name: ci
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - '**'
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - run: yarn
+    - run: yarn lint
+    - uses: gozala/[email protected]
+    - run: yarn build
+    - run: yarn aegir dep-check
+    - uses: ipfs/aegir/actions/bundle-size@master
+      name: size
+      with:
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+  test-node:
+    needs: check
+    runs-on: ${{ matrix.os }}
+    services:
+      mysql:
+        image: mysql:5.7
+        env:
+          MYSQL_ROOT_PASSWORD: test-secret-pw
+          MYSQL_DATABASE: libp2p_rendezvous_db
+        ports:
+        - 3306:3306
+        options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        node: [12, 14]
+      fail-fast: true
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v1
+        with:
+          node-version: ${{ matrix.node }}
+      - run: yarn
+      - run: npx nyc --reporter=lcov aegir test -t node -- --bail
+      - uses: codecov/codecov-action@v1
+  test-chrome:
+    needs: check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - run: yarn
+      - run: npx aegir test -t browser -t webworker --bail
+  test-firefox:
+    needs: check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - run: yarn
+      - run: npx aegir test -t browser -t webworker --bail -- --browsers FirefoxHeadless

.travis.yml

@@ -4,8 +4,10 @@
 [![](https://img.shields.io/badge/project-libp2p-yellow.svg?style=flat-square)](http://libp2p.io/)
 [![](https://img.shields.io/badge/freenode-%23libp2p-yellow.svg?style=flat-square)](http://webchat.freenode.net/?channels=%23libp2p)
 [![](https://img.shields.io/discourse/https/discuss.libp2p.io/posts.svg)](https://discuss.libp2p.io)
+[![codecov](https://img.shields.io/codecov/c/github/libp2p/js-libp2p-rendezvous.svg?style=flat-square)](https://codecov.io/gh/libp2p/js-libp2p-rendezvous)
+[![GitHub Workflow Status](https://img.shields.io/github/workflow/status/libp2p/js-libp2p-rendezvous/ci?label=ci&style=flat-square)](https://github.com/libp2p/js-libp2p-rendezvous/actions?query=branch%3Amaster+workflow%3Aci+)
 
-> Javascript implementation of the rendezvous protocol for libp2p
+> Javascript implementation of the rendezvous server protocol for libp2p
 
 ## Lead Maintainer <!-- omit in toc -->
 
@@ -23,7 +25,7 @@
 
 ## Overview
 
-Libp2p rendezvous is a lightweight mechanism for generalized peer discovery. It can be used for bootstrap purposes, real time peer discovery, application specific routing, and so on. Any node implementing the rendezvous protocol can act as a rendezvous point, allowing the discovery of relevant peers in a decentralized fashion.
+Libp2p rendezvous is a lightweight mechanism for generalized peer discovery. It can be used for bootstrap purposes, real time peer discovery, application specific routing, and so on. This module is the implementation of the rendezvous server protocol for libp2p.
 
 See the [SPEC](https://github.com/libp2p/specs/tree/master/rendezvous) for more details.
 
@@ -35,35 +37,58 @@ See the [SPEC](https://github.com/libp2p/specs/tree/master/rendezvous) for more
 > npm install --global libp2p-rendezvous
 ```
 
-Now you can use the cli command `libp2p-rendezvous-server` to spawn a libp2p rendezvous server.
+Now you can use the cli command `libp2p-rendezvous-server` to spawn a libp2p rendezvous server. Bear in mind that a MySQL database is required to run the rendezvous server.
+
+### Testing
+
+For running the tests in this module, you will need to have Docker installed. A docker container is used to run a MySQL database for testing purposes.
 
 ### CLI
 
-After installing the rendezvous server, you can use its binary. It accepts several arguments: `--peerId`, `--listenMultiaddrs`, `--announceMultiaddrs`, `--metricsPort` and `--disableMetrics`
+After installing the rendezvous server, you can use its binary. It accepts several arguments: `--datastoreHost`, `--datastoreUser`, `--datastorePassword`, `--datastoreDatabase`, `--enableMemoryDatabase`, `--peerId`, `--listenMultiaddrs`, `--announceMultiaddrs`, `--metricsPort` and `--disableMetrics`
 
 ```sh
-libp2p-rendezvous-server [--peerId <jsonFilePath>] [--listenMultiaddrs <ma> ... <ma>] [--announceMultiaddrs <ma> ... <ma>] [--metricsPort <port>] [--disableMetrics]
+libp2p-rendezvous-server [--datastoreHost <hostname>] [--datastoreUser <username>] [datastorePassword <password>] [datastoreDatabase <name>] [--enableMemoryDatabase] [--peerId <jsonFilePath>] [--listenMultiaddrs <ma> ... <ma>] [--announceMultiaddrs <ma> ... <ma>] [--metricsPort <port>] [--disableMetrics]
 ```
 
-For further customization (e.g. swapping the muxer, using other transports) it is recommended to create a server via the API.
+For further customization (e.g. swapping the muxer, using other transports, use other database) it is recommended to create a server via the API.
+
+#### Datastore
+
+A rendezvous server needs to leverage a MySQL database as a datastore for the registrations. This needs to be configured in order to run a rendezvous server. You can rely on docker to run a MySQL database using a command like:
+
+```sh
+docker run -p 3306:3306 -e MYSQL_ROOT_PASSWORD=your-secret-pw -e MYSQL_DATABASE=libp2p_rendezvous_db -d mysql:8 --default-authentication-plugin=mysql_native_password
+```
+
+Once a MySQL database is running, you can run the rendezvous server by providing the datastore configuration options as follows:
+
+```sh
+libp2p-rendezvous-server --datastoreHost 'localhost' --datastoreUser 'root' --datastorePassword 'your-secret-pw' --datastoreDatabase 'libp2p_rendezvous_db'
+```
+
+⚠️ For testing purposes you can skip using MySQL and use a memory datastore. This must not be used in production! For this you just need to provide the `--enableMemoryDatabase` option.
 
 #### PeerId
 
-You can create a [PeerId](https://github.com/libp2p/js-peer-id) via its [CLI](https://github.com/libp2p/js-peer-id#cli). 
+You can create a [PeerId](https://github.com/libp2p/js-peer-id) via its [CLI](https://github.com/libp2p/js-peer-id#cli) and use it in the rendezvous server.
+
+Once you have a generated PeerId json file, you can start the rendezvous with that PeerId by specifying its path via the `--peerId` flag:
 
 ```sh
-libp2p-rendezvous-server --peerId id.json
+peer-id --type=ed25519 > id.json
+libp2p-rendezvous-server --peerId id.json --datastoreHost 'localhost' --datastoreUser 'root' --datastorePassword 'your-secret-pw' --datastoreDatabase 'libp2p_rendezvous_db'
 ```
 
 #### Multiaddrs
 
 You can specify the libp2p rendezvous server listen and announce multiaddrs. This server is configured with [libp2p-tcp](https://github.com/libp2p/js-libp2p-tcp) and [libp2p-websockets](https://github.com/libp2p/js-libp2p-websockets) and addresses with this transports should be used. It can always be modified via the API.
 
 ```sh
-libp2p-rendezvous-server --peerId id.json --listenMultiaddrs '/ip4/127.0.0.1/tcp/15002/ws' '/ip4/127.0.0.1/tcp/8000' --announceMultiaddrs '/dns4/test.io/tcp/443/wss/p2p/12D3KooWAuEpJKhCAfNcHycKcZCv9Qy69utLAJ3MobjKpsoKbrGA' '/dns6/test.io/tcp/443/wss/p2p/12D3KooWAuEpJKhCAfNcHycKcZCv9Qy69utLAJ3MobjKpsoKbrGA'
+libp2p-rendezvous-server --peerId id.json --listenMultiaddrs '/ip4/127.0.0.1/tcp/15002/ws' '/ip4/127.0.0.1/tcp/8000' --announceMultiaddrs '/dns4/test.io/tcp/443/wss/p2p/12D3KooWAuEpJKhCAfNcHycKcZCv9Qy69utLAJ3MobjKpsoKbrGA' '/dns6/test.io/tcp/443/wss/p2p/12D3KooWAuEpJKhCAfNcHycKcZCv9Qy69utLAJ3MobjKpsoKbrGA' --datastoreHost 'localhost' --datastoreUser 'root' --datastorePassword 'your-secret-pw' --datastoreDatabase 'libp2p_rendezvous_db'
 ```
 
-By default it listens on `/ip4/127.0.0.1/tcp/15002/ws` and has no announce multiaddrs specified.
+By default it listens on `/ip4/127.0.0.1/tcp/8000` and `/ip4/127.0.0.1/tcp/15003/ws`. It has no announce multiaddrs specified.
 
 #### Metrics
 
@@ -81,8 +106,30 @@ libp2p-rendezvous-server --disableMetrics
 
 ### Docker Setup
 
+```yml
+version: '3.1'
+services:
+  db:
+    image: mysql
+    volumes:
+        - mysql-db:/var/lib/mysql
+    command: --default-authentication-plugin=mysql_native_password
+    restart: always
+    environment:
+      MYSQL_ROOT_PASSWORD: your-secret-pw
+      MYSQL_DATABASE: libp2p_rendezvous_db
+    ports:
+      - "3306:3306"
+volumes:
+  mysql-db:
+```
+
+## Library
+
 TODO
 
+Datastores
+
 ## Contribute
 
 Feel free to join in. All welcome. Open an [issue](https://github.com/libp2p/js-libp2p-rendezvous/issues)!

README.md

@@ -7,14 +7,14 @@ services:
     command: --default-authentication-plugin=mysql_native_password
     restart: always
     environment:
-      # MYSQL_ROOT_PASSWORD: my-secret-pw
-      # MYSQL_USER: libp2p
-      # MYSQL_PASSWORD: my-secret-pw
-      # MYSQL_DATABASE: libp2p_rendezvous_db
-      MYSQL_DATABASE: ${DATABASE}
-      MYSQL_ROOT_PASSWORD: ${ROOT_PASSWORD}
-      MYSQL_USER: ${USER}
-      MYSQL_PASSWORD: ${PASSWORD}
+      MYSQL_ROOT_PASSWORD: my-secret-pw
+      MYSQL_USER: libp2p
+      MYSQL_PASSWORD: my-secret-pw
+      MYSQL_DATABASE: libp2p_rendezvous_db
+      # MYSQL_DATABASE: ${DATABASE}
+      # MYSQL_ROOT_PASSWORD: ${ROOT_PASSWORD}
+      # MYSQL_USER: ${USER}
+      # MYSQL_PASSWORD: ${PASSWORD}
     ports:
       - "3306:3306"
 volumes:

mysql/Dockerfile

@@ -30,6 +30,9 @@
     "node": ">=12.0.0",
     "npm": ">=6.0.0"
   },
+  "browser": {
+    "mysql": false
+  },
   "scripts": {
     "lint": "aegir lint",
     "build": "aegir build",
@@ -48,7 +51,7 @@
     "it-buffer": "^0.1.2",
     "it-length-prefixed": "^3.1.0",
     "it-pipe": "^1.1.0",
-    "libp2p": "libp2p/js-libp2p#chore/add-typedfs-with-post-install",
+    "libp2p": "^0.30.0",
     "libp2p-mplex": "^0.10.0",
     "libp2p-noise": "^2.0.1",
     "libp2p-tcp": "^0.15.1",
@@ -59,14 +62,18 @@
     "mysql": "^2.18.1",
     "peer-id": "^0.14.1",
     "protons": "^2.0.0",
-    "streaming-iterables": "^5.0.2"
+    "streaming-iterables": "^5.0.2",
+    "uint8arrays": "^2.0.5"
   },
   "devDependencies": {
     "aegir": "^29.2.2",
     "chai": "^4.2.0",
     "chai-as-promised": "^7.1.1",
     "delay": "^4.4.0",
     "dirty-chai": "^2.0.1",
+    "execa": "^5.0.0",
+    "ipfs-utils": "^5.0.1",
+    "is-ci": "^2.0.0",
     "p-defer": "^3.0.0",
     "p-times": "^3.0.0",
     "p-wait-for": "^3.1.0",

mysql/docker-compose.yml

@@ -0,0 +1,27 @@
+USE libp2p_rendezvous_db
+
+CREATE TABLE IF NOT EXISTS registration (
+  id INT UNSIGNED NOT NULL AUTO_INCREMENT,
+  namespace varchar(255) NOT NULL,
+  peer_id varchar(255) NOT NULL,
+  PRIMARY KEY (id),
+  INDEX (namespace, peer_id)
+);
+
+CREATE TABLE IF NOT EXISTS cookie (
+  id varchar(21),
+  namespace varchar(255),
+  reg_id INT UNSIGNED,
+  peer_id varchar(255) NOT NULL,
+  created_at datetime DEFAULT CURRENT_TIMESTAMP,
+  PRIMARY KEY (id, namespace, reg_id),
+  INDEX (created_at)
+);
+
+INSERT INTO registration (namespace, peer_id) VALUES ('test-ns', 'QmW8rAgaaA6sRydK1k6vonShQME47aDxaFidbtMevWs73t');
+
+SELECT * FROM registration
+
+SELECT * FROM cookie
+
+INSERT INTO registration (namespace, peer_id) VALUES ('test-ns', 'QmZqCdSzgpsmB3Qweb9s4fojAoqELWzqku21UVrqtVSKi4');