feat: use new datastore-idb (#225)

hugomrdias · achingbrain · web-flow · commit 99df42bbf4bd · 2020-04-20T13:00:14.000+01:00
A new fast and smaller browser datastore is used `datastore-idb` https://github.com/ipfs/js-datastore-idb Also removes node globals and reduces bundle size. size: 77.09KB -> 45.07KB ``` datastore-idb up to 44% faster and 69% smaller batch idb x 5.26 ops/sec ±4.57% (29 runs sampled) batch level x 2.92 ops/sec ±3.76% (19 runs sampled) Fastest is batch idb size 37.32KB to 11.41KB ``` Co-authored-by: Alex Potsides <alex@achingbrain.net>
diff --git a/.travis.yml b/.travis.yml
@@ -6,8 +6,8 @@ stages:
   - cov
 
 node_js:
-  - '10'
   - '12'
+  - '10'
 
 os:
   - linux
diff --git a/README.md b/README.md
@@ -181,21 +181,21 @@ Get a value at the root of the repo.
 
 * `key` can be a buffer, a string or a [Key](https://github.com/ipfs/interface-datastore#keys).
 
-[Blocks](https://github.com/ipfs/js-ipfs-block#readme):
+[Blocks](https://github.com/ipfs/js-ipld-block#readme):
 
 #### `Promise<Boolean> repo.isInitialized ()`
 
 The returned promise resolves to `false` if the repo has not been initialized and `true` if it has.
 
 #### `Promise repo.blocks.put (block:Block)`
 
-* `block` should be of type [Block](https://github.com/ipfs/js-ipfs-block#readme).
+* `block` should be of type [Block](https://github.com/ipfs/js-ipld-block#readme).
 
 #### `Promise repo.blocks.putMany (blocks)`
 
 Put many blocks.
 
-* `block` should be an Iterable or AsyncIterable that yields entries of type [Block](https://github.com/ipfs/js-ipfs-block#readme).
+* `block` should be an Iterable or AsyncIterable that yields entries of type [Block](https://github.com/ipfs/js-ipld-block#readme).
 
 #### `Promise<Buffer> repo.blocks.get (cid)`
 
diff --git a/package.json b/package.json
@@ -10,7 +10,7 @@
   ],
   "browser": {
     "rimraf": false,
-    "datastore-fs": "datastore-level",
+    "datastore-fs": "datastore-idb",
     "./src/lock.js": "./src/lock-memory.js",
     "./src/default-options.js": "./src/default-options-browser.js"
   },
@@ -43,33 +43,35 @@
     "npm": ">=3.0.0"
   },
   "devDependencies": {
-    "aegir": "^21.4.5",
+    "aegir": "^21.8.1",
     "chai": "^4.2.0",
     "dirty-chai": "^2.0.1",
-    "lodash": "^4.17.11",
+    "just-range": "^2.1.0",
     "memdown": "^5.1.0",
     "multihashes": "~0.4.15",
     "multihashing-async": "~0.8.0",
     "ncp": "^2.0.0",
     "rimraf": "^3.0.0",
-    "sinon": "^9.0.1"
+    "sinon": "^9.0.2"
   },
   "dependencies": {
-    "base32.js": "~0.1.0",
     "bignumber.js": "^9.0.0",
+    "buffer": "^5.5.0",
     "bytes": "^3.1.0",
     "cids": "^0.8.0",
-    "datastore-core": "~0.7.0",
+    "datastore-core": "^1.0.0",
     "datastore-fs": "~0.9.0",
+    "datastore-idb": "^1.0.0",
     "datastore-level": "~0.14.0",
     "debug": "^4.1.0",
     "err-code": "^2.0.0",
-    "interface-datastore": "^0.8.0",
-    "ipfs-block": "~0.8.1",
-    "ipfs-repo-migrations": "~0.1.0",
+    "interface-datastore": "^0.8.3",
+    "ipfs-repo-migrations": "^0.2.0",
+    "ipfs-utils": "^2.2.0",
+    "ipld-block": "^0.9.1",
     "just-safe-get": "^2.0.0",
     "just-safe-set": "^2.1.0",
-    "lodash.has": "^4.5.2",
+    "multibase": "^0.7.0",
     "p-queue": "^6.0.0",
     "proper-lockfile": "^4.0.0",
     "sort-keys": "^4.0.0"
diff --git a/src/api-addr.js b/src/api-addr.js
@@ -1,5 +1,6 @@
 'use strict'
 
+const { Buffer } = require('buffer')
 const Key = require('interface-datastore').Key
 
 const apiFile = new Key('api')
diff --git a/src/blockstore-utils.js b/src/blockstore-utils.js
@@ -1,8 +1,8 @@
 'use strict'
 
-const base32 = require('base32.js')
 const { Key } = require('interface-datastore')
 const CID = require('cids')
+const multibase = require('multibase')
 
 /**
  * Transform a cid to the appropriate datastore key.
@@ -11,8 +11,7 @@ const CID = require('cids')
  * @returns {Key}
  */
 exports.cidToKey = cid => {
-  const enc = new base32.Encoder()
-  return new Key('/' + enc.write(cid.buffer).finalize(), false)
+  return new Key('/' + multibase.encode('base32', cid.buffer).toString().slice(1).toUpperCase(), false)
 }
 
 /**
@@ -22,8 +21,5 @@ exports.cidToKey = cid => {
  * @returns {CID}
  */
 exports.keyToCid = key => {
-  // Block key is of the form /<base32 encoded string>
-  const decoder = new base32.Decoder()
-  const buff = decoder.write(key.toString().slice(1)).finalize()
-  return new CID(Buffer.from(buff))
+  return new CID(multibase.decode('b' + key.toString().slice(1).toLowerCase()))
 }
diff --git a/src/blockstore.js b/src/blockstore.js
@@ -2,7 +2,7 @@
 
 const core = require('datastore-core')
 const ShardingStore = core.ShardingDatastore
-const Block = require('ipfs-block')
+const Block = require('ipld-block')
 const CID = require('cids')
 const errcode = require('err-code')
 const { cidToKey } = require('./blockstore-utils')
diff --git a/src/config.js b/src/config.js
@@ -1,10 +1,10 @@
 'use strict'
 
+const { Buffer } = require('buffer')
 const Key = require('interface-datastore').Key
 const { default: Queue } = require('p-queue')
 const _get = require('just-safe-get')
 const _set = require('just-safe-set')
-const _has = require('lodash.has')
 const errcode = require('err-code')
 const errors = require('./errors')
 
@@ -27,7 +27,7 @@ module.exports = (store) => {
 
       const encodedValue = await store.get(configKey)
       const config = JSON.parse(encodedValue.toString())
-      if (key !== undefined && !_has(config, key)) {
+      if (key !== undefined && _get(config, key) === undefined) {
         throw new errors.NotFoundError(`Key ${key} does not exist in config`)
       }
 
diff --git a/src/default-options-browser.js b/src/default-options-browser.js
@@ -4,10 +4,10 @@
 module.exports = {
   lock: 'memory',
   storageBackends: {
-    root: require('datastore-level'),
-    blocks: require('datastore-level'),
-    keys: require('datastore-level'),
-    datastore: require('datastore-level')
+    root: require('datastore-idb'),
+    blocks: require('datastore-idb'),
+    keys: require('datastore-idb'),
+    datastore: require('datastore-idb')
   },
   storageBackendOptions: {
     root: {
diff --git a/src/index.js b/src/index.js
@@ -1,13 +1,12 @@
 'use strict'
 
 const _get = require('just-safe-get')
-const assert = require('assert')
-const path = require('path')
 const debug = require('debug')
 const Big = require('bignumber.js')
 const errcode = require('err-code')
 const migrator = require('ipfs-repo-migrations')
 const bytes = require('bytes')
+const pathJoin = require('ipfs-utils/src/path-join')
 
 const constants = require('./constants')
 const backends = require('./backends')
@@ -40,7 +39,9 @@ class IpfsRepo {
    * @param {object} options - Configuration
    */
   constructor (repoPath, options) {
-    assert.strictEqual(typeof repoPath, 'string', 'missing repoPath')
+    if (typeof repoPath !== 'string') {
+      throw new Error('missing repoPath')
+    }
 
     this.options = buildOptions(options)
     this.closed = true
@@ -112,13 +113,15 @@ class IpfsRepo {
       this.lockfile = await this._openLock(this.path)
       log('acquired repo.lock')
       log('creating datastore')
-      this.datastore = backends.create('datastore', path.join(this.path, 'datastore'), this.options)
+      this.datastore = backends.create('datastore', pathJoin(this.path, 'datastore'), this.options)
+      await this.datastore.open()
       log('creating blocks')
-      const blocksBaseStore = backends.create('blocks', path.join(this.path, 'blocks'), this.options)
+      const blocksBaseStore = backends.create('blocks', pathJoin(this.path, 'blocks'), this.options)
+      await blocksBaseStore.open()
       this.blocks = await blockstore(blocksBaseStore, this.options.storageBackendOptions.blocks)
       log('creating keystore')
-      this.keys = backends.create('keys', path.join(this.path, 'keys'), this.options)
-
+      this.keys = backends.create('keys', pathJoin(this.path, 'keys'), this.options)
+      await this.keys.open()
       const isCompatible = await this.version.check(constants.repoVersion)
       if (!isCompatible) {
         if (await this._isAutoMigrationEnabled()) {
@@ -152,11 +155,15 @@ class IpfsRepo {
    */
   _getLocker () {
     if (typeof this.options.lock === 'string') {
-      assert(lockers[this.options.lock], 'Unknown lock type: ' + this.options.lock)
+      if (!lockers[this.options.lock]) {
+        throw new Error('Unknown lock type: ' + this.options.lock)
+      }
       return lockers[this.options.lock]
     }
 
-    assert(this.options.lock, 'No lock provided')
+    if (!this.options.lock) {
+      throw new Error('No lock provided')
+    }
     return this.options.lock
   }
 
@@ -251,7 +258,13 @@ class IpfsRepo {
       }
     }
 
-    await Promise.all([this.root, this.blocks, this.keys, this.datastore].map((store) => store.close()))
+    await Promise.all([
+      this.root,
+      this.blocks,
+      this.keys,
+      this.datastore
+    ].map((store) => store.close()))
+
     log('unlocking')
     this.closed = true
     await this._closeLock()
diff --git a/src/lock.js b/src/lock.js
@@ -1,5 +1,6 @@
 'use strict'
 
+const { LockExistsError } = require('./errors')
 const path = require('path')
 const debug = require('debug')
 const { lock } = require('proper-lockfile')
@@ -27,7 +28,16 @@ const STALE_TIME = 20000
 exports.lock = async (dir) => {
   const file = path.join(dir, lockFile)
   log('locking %s', file)
-  const release = await lock(dir, { lockfilePath: file, stale: STALE_TIME })
+  let release
+  try {
+    release = await lock(dir, { lockfilePath: file, stale: STALE_TIME })
+  } catch (err) {
+    if (err.code === 'ELOCKED') {
+      throw new LockExistsError(`Lock already being held for file: ${file}`)
+    } else {
+      throw err
+    }
+  }
   return {
     close: async () => { // eslint-disable-line require-await
       release()
diff --git a/src/spec.js b/src/spec.js
@@ -1,5 +1,6 @@
 'use strict'
 
+const { Buffer } = require('buffer')
 const Key = require('interface-datastore').Key
 const sortKeys = require('sort-keys')
 
diff --git a/src/version.js b/src/version.js
@@ -1,5 +1,6 @@
 'use strict'
 
+const { Buffer } = require('buffer')
 const Key = require('interface-datastore').Key
 const debug = require('debug')
 const log = debug('repo:version')
diff --git a/test/api-addr-test.js b/test/api-addr-test.js
@@ -1,6 +1,7 @@
 /* eslint-env mocha */
 'use strict'
 
+const { Buffer } = require('buffer')
 const chai = require('chai')
 chai.use(require('dirty-chai'))
 const expect = chai.expect
diff --git a/test/blockstore-test.js b/test/blockstore-test.js
@@ -2,22 +2,22 @@
 /* eslint-env mocha */
 'use strict'
 
+const { Buffer } = require('buffer')
 const chai = require('chai')
 chai.use(require('dirty-chai'))
 const expect = chai.expect
 const assert = chai.assert
-const Block = require('ipfs-block')
+const Block = require('ipld-block')
 const CID = require('cids')
-const _ = require('lodash')
+const range = require('just-range')
 const multihashing = require('multihashing-async')
-const path = require('path')
-const Key = require('interface-datastore').Key
-const base32 = require('base32.js')
+const tempDir = require('ipfs-utils/src/temp-dir')
+const { cidToKey } = require('../src/blockstore-utils')
 const IPFSRepo = require('../')
 
 module.exports = (repo) => {
   describe('blockstore', () => {
-    const blockData = _.range(100).map((i) => Buffer.from(`hello-${i}-${Math.random()}`))
+    const blockData = range(100).map((i) => Buffer.from(`hello-${i}-${Math.random()}`))
     const bData = Buffer.from('hello world')
     let b
 
@@ -52,16 +52,16 @@ module.exports = (repo) => {
 
       it('massive multiwrite', async function () {
         this.timeout(15000) // add time for ci
-        const hashes = await Promise.all(_.range(100).map((i) => multihashing(blockData[i], 'sha2-256')))
-        await Promise.all(_.range(100).map((i) => {
+        const hashes = await Promise.all(range(100).map((i) => multihashing(blockData[i], 'sha2-256')))
+        await Promise.all(range(100).map((i) => {
           const block = new Block(blockData[i], new CID(hashes[i]))
           return repo.blocks.put(block)
         }))
       })
 
       it('.putMany', async function () {
         this.timeout(15000) // add time for ci
-        const blocks = await Promise.all(_.range(50).map(async (i) => {
+        const blocks = await Promise.all(range(50).map(async (i) => {
           const d = Buffer.from('many' + Math.random())
           const hash = await multihashing(d, 'sha2-256')
           return new Block(d, new CID(hash))
@@ -79,9 +79,12 @@ module.exports = (repo) => {
         const cid = new CID(hash)
         let putInvoked = false
         let commitInvoked = false
-        otherRepo = new IPFSRepo(path.join(path.basename(repo.path), '/repo-' + Date.now()), {
+        otherRepo = new IPFSRepo(tempDir(), {
           storageBackends: {
             blocks: class ExplodingBlockStore {
+              open () {
+              }
+
               close () {
 
               }
@@ -147,7 +150,7 @@ module.exports = (repo) => {
 
       it('massive read', async function () {
         this.timeout(15000) // add time for ci
-        await Promise.all(_.range(20 * 100).map(async (i) => {
+        await Promise.all(range(20 * 100).map(async (i) => {
           const j = i % blockData.length
           const hash = await multihashing(blockData[j], 'sha2-256')
           const block = await repo.blocks.get(new CID(hash))
@@ -210,12 +213,12 @@ module.exports = (repo) => {
         const data = Buffer.from(`TEST${Date.now()}`)
         const hash = await multihashing(data, 'sha2-256')
         const cid = new CID(hash)
-        const enc = new base32.Encoder()
-        const key = new Key('/' + enc.write(cid.buffer).finalize(), false)
+        const key = cidToKey(cid)
 
-        otherRepo = new IPFSRepo(path.join(path.basename(repo.path), '/repo-' + Date.now()), {
+        otherRepo = new IPFSRepo(tempDir(), {
           storageBackends: {
             blocks: class ExplodingBlockStore {
+              open () {}
               close () {
 
               }
diff --git a/test/config-test.js b/test/config-test.js
@@ -1,6 +1,7 @@
 /* eslint-env mocha */
 'use strict'
 
+const { Buffer } = require('buffer')
 const chai = require('chai')
 chai.use(require('dirty-chai'))
 const expect = chai.expect
diff --git a/test/datastore-test.js b/test/datastore-test.js
diff --git a/test/is-initialized.js b/test/is-initialized.js
diff --git a/test/lock-test.js b/test/lock-test.js
diff --git a/test/node.js b/test/node.js
diff --git a/test/options-test.js b/test/options-test.js
diff --git a/test/repo-test.js b/test/repo-test.js
diff --git a/test/stat-test.js b/test/stat-test.js

-Original file line number
+Diff line change
   - cov
 node_js:
 -  - '10'
   - '12'
 +  - '10'
 os:
   - linux