feat: split .query into .query and .queryKeys (#87)

The return type of the `.query` function varies depending on whether
`keysOnly` is passed as part of the query params.  This makes mapping
the return type difficult as it's dependent on the function input.

Here we:

1. Constrain the return type of `.query` to just key/value pairs
1. Add a new `.queryKeys` method that returns only keys
1. Add the implementation in the adapter to just chain to the existing `.query` method and map the result
1. Remove redundant utils
1. Remove redundant TextEncoder/Decoder exports
1. Change the query sort method to be a regular js sort function
1. Export types for query sort/filter methods

Co-authored-by: Vasco Santos <[email protected]>

Author: achingbrain

.github/workflows/main.yml

@@ -35,7 +35,7 @@ jobs:
         with:
           node-version: ${{ matrix.node }}
       - run: npm install
-      - run: npx nyc --reporter=lcov aegir test -t node -- --bail
+      - run: npx aegir test -t node --cov --bail
       - uses: codecov/codecov-action@v1
   test-chrome:
     needs: check
@@ -50,7 +50,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - run: npm install
-      - run: npx aegir test -t browser -t webworker --bail -- --browsers FirefoxHeadless
+      - run: npx aegir test -t browser -t webworker --bail -- --browser firefox
   test-electron-main:
     needs: check
     runs-on: ubuntu-latest

package.json

@@ -10,16 +10,15 @@
     "dist"
   ],
   "scripts": {
-    "prepare": "aegir build --no-bundle && cp src/types.d.ts dist/src/types.d.ts",
+    "prepare": "aegir build --no-bundle",
     "lint": "aegir lint",
     "test": "aegir test",
     "test:node": "aegir test --target node",
     "test:browser": "aegir test --target browser",
     "release": "aegir release --docs",
     "release-minor": "aegir release --type minor --docs",
     "release-major": "aegir release --type major --docs",
-    "coverage": "aegir coverage",
-    "coverage-publish": "aegir coverage --provider codecov",
+    "coverage": "aegir test --cov",
     "docs": "aegir docs"
   },
   "repository": {
@@ -39,15 +38,19 @@
   "homepage": "https://github.com/ipfs/interface-datastore#readme",
   "devDependencies": {
     "@types/err-code": "^2.0.0",
-    "aegir": "^33.1.0"
+    "aegir": "^33.1.0",
+    "it-map": "^1.0.5"
   },
   "dependencies": {
     "err-code": "^3.0.1",
     "ipfs-utils": "^6.0.0",
     "iso-random-stream": "^2.0.0",
     "it-all": "^1.0.2",
     "it-drain": "^1.0.1",
-    "nanoid": "^3.0.2"
+    "it-filter": "^1.0.2",
+    "it-take": "^1.0.1",
+    "nanoid": "^3.0.2",
+    "uint8arrays": "^2.1.5"
   },
   "eslintConfig": {
     "extends": "ipfs"

src/adapter.js

@@ -1,14 +1,17 @@
 'use strict'
 
-const { filter, sortAll, take, map } = require('./utils')
+const { sortAll } = require('./utils')
 const drain = require('it-drain')
+const filter = require('it-filter')
+const take = require('it-take')
 
 /**
  * @typedef {import('./key')} Key
  * @typedef {import('./types').Pair} Pair
  * @typedef {import('./types').Datastore} Datastore
  * @typedef {import('./types').Options} Options
  * @typedef {import('./types').Query} Query
+ * @typedef {import('./types').KeyQuery} KeyQuery
  * @typedef {import('./types').Batch} Batch
  */
 
@@ -134,6 +137,8 @@ class Adapter {
   }
 
   /**
+   * Extending classes should override `query` or implement this method
+   *
    * @param {Query} q
    * @param {Options} [options]
    * @returns {AsyncIterable<Pair>}
@@ -143,6 +148,18 @@ class Adapter {
     throw new Error('._all is not implemented')
   }
 
+  /**
+   * Extending classes should override `queryKeys` or implement this method
+   *
+   * @param {KeyQuery} q
+   * @param {Options} [options]
+   * @returns {AsyncIterable<Key>}
+   */
+  // eslint-disable-next-line require-yield
+  async * _allKeys (q, options) {
+    throw new Error('._allKeys is not implemented')
+  }
+
   /**
    * @param {Query} q
    * @param {Options} [options]
@@ -173,8 +190,37 @@ class Adapter {
       it = take(it, q.limit)
     }
 
-    if (q.keysOnly === true) {
-      return map(it, (e) => /** @type {Pair} */({ key: e.key }))
+    return it
+  }
+
+  /**
+   * @param {KeyQuery} q
+   * @param {Options} [options]
+   */
+  queryKeys (q, options) {
+    let it = this._allKeys(q, options)
+
+    if (q.prefix != null) {
+      it = filter(it, (key) =>
+        key.toString().startsWith(/** @type {string} */ (q.prefix))
+      )
+    }
+
+    if (Array.isArray(q.filters)) {
+      it = q.filters.reduce((it, f) => filter(it, f), it)
+    }
+
+    if (Array.isArray(q.orders)) {
+      it = q.orders.reduce((it, f) => sortAll(it, f), it)
+    }
+
+    if (q.offset != null) {
+      let i = 0
+      it = filter(it, () => i++ >= /** @type {number} */ (q.offset))
+    }
+
+    if (q.limit != null) {
+      it = take(it, q.limit)
     }
 
     return it

src/index.js

@@ -5,6 +5,11 @@
  * @typedef {import('./types').Batch} Batch
  * @typedef {import('./types').Options} Options
  * @typedef {import('./types').Query} Query
+ * @typedef {import('./types').QueryFilter} QueryFilter
+ * @typedef {import('./types').QueryOrder} QueryOrder
+ * @typedef {import('./types').KeyQuery} KeyQuery
+ * @typedef {import('./types').KeyQueryFilter} KeyQueryFilter
+ * @typedef {import('./types').KeyQueryOrder} KeyQueryOrder
  * @typedef {import('./types').Pair} Pair
  */
 

src/key.js

@@ -1,11 +1,13 @@
 'use strict'
 
 const { nanoid } = require('nanoid')
-const { utf8Encoder, utf8Decoder } = require('./utils')
+
+const uint8ArrayToString = require('uint8arrays/to-string')
+const uint8ArrayFromString = require('uint8arrays/from-string')
 
 const symbol = Symbol.for('@ipfs/interface-datastore/key')
 const pathSepS = '/'
-const pathSepB = utf8Encoder.encode(pathSepS)
+const pathSepB = new TextEncoder().encode(pathSepS)
 const pathSep = pathSepB[0]
 
 /**
@@ -31,7 +33,7 @@ class Key {
    */
   constructor (s, clean) {
     if (typeof s === 'string') {
-      this._buf = utf8Encoder.encode(s)
+      this._buf = uint8ArrayFromString(s)
     } else if (s instanceof Uint8Array) {
       this._buf = s
     } else {
@@ -54,15 +56,11 @@ class Key {
   /**
    * Convert to the string representation
    *
-   * @param {string} [encoding='utf8'] - The encoding to use.
+   * @param {import('uint8arrays/to-string').SupportedEncodings} [encoding='utf8'] - The encoding to use.
    * @returns {string}
    */
   toString (encoding = 'utf8') {
-    if (encoding === 'utf8' || encoding === 'utf-8') {
-      return utf8Decoder.decode(this._buf)
-    }
-
-    return new TextDecoder(encoding).decode(this._buf)
+    return uint8ArrayToString(this._buf, encoding)
   }
 
   /**

src/memory.js

@@ -65,6 +65,11 @@ class MemoryDatastore extends Adapter {
     yield * Object.entries(this.data)
       .map(([key, value]) => ({ key: new Key(key), value }))
   }
+
+  async * _allKeys () {
+    yield * Object.entries(this.data)
+      .map(([key]) => new Key(key))
+  }
 }
 
 module.exports = MemoryDatastore