From 1c00d9def70ea881f76322391d52ce34b51eb300 Mon Sep 17 00:00:00 2001 From: Hugo Dias Date: Tue, 16 Jun 2020 12:25:44 +0100 Subject: [PATCH 1/6] fix: fix undefined name with prefix --- src/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/index.js b/src/index.js index fadb011..10a0558 100644 --- a/src/index.js +++ b/src/index.js @@ -76,7 +76,7 @@ class IdbDatastore extends Adapter { this.store = null this.options = options - this.location = options.prefix + location + this.location = (options.prefix || '') + location this.version = options.version || 1 } From b5072d41a78e328eb7abbb0c29cfdcadc6d18e35 Mon Sep 17 00:00:00 2001 From: Hugo Dias Date: Tue, 16 Jun 2020 12:26:25 +0100 Subject: [PATCH 2/6] fix: re use query tx for ops --- src/index.js | 78 ++++++++++++++++++++++++++++++++-------------------- 1 file changed, 48 insertions(+), 30 deletions(-) diff --git a/src/index.js b/src/index.js index 10a0558..0e40633 100644 --- a/src/index.js +++ b/src/index.js @@ -43,11 +43,16 @@ const str2ab = (str) => { return buf } -const queryIt = async function * (q, store, location) { +const queryIt = async function * (q, instance) { + const { db, location } = instance const range = q.prefix ? self.IDBKeyRange.bound(str2ab(q.prefix), str2ab(q.prefix + '\xFF'), false, true) : undefined - let cursor = await store.transaction(location).store.openCursor(range) + const tx = db.transaction(location, 'readwrite') + const store = tx.objectStore(location) + let cursor = await store.openCursor(range) let limit = 0 + instance.tx = tx + if (cursor && q.offset && q.offset > 0) { cursor = await cursor.advance(q.offset) } @@ -68,26 +73,41 @@ const queryIt = async function * (q, store, location) { } cursor = await cursor.continue() } + instance.tx = null } class IdbDatastore extends Adapter { constructor (location, options = {}) { super() - this.store = null + this.db = null this.options = options this.location = (options.prefix || '') + location this.version = options.version || 1 + /** @type {IDBTransaction} */ + this.tx = null + } + + getStore (mode) { + if (this.db === null) { + throw new Error('Datastore needs to be opened.') + } + + if (this.tx) { + return this.tx.objectStore(this.location) + } + + return this.db.transaction(this.location, mode).objectStore(this.location) } async open () { - if (this.store !== null) { + if (this.db !== null) { return } const location = this.location try { - this.store = await openDB(this.location, this.version, { + this.db = await openDB(this.location, this.version, { upgrade (db) { db.createObjectStore(location) } @@ -98,23 +118,17 @@ class IdbDatastore extends Adapter { } async put (key, val) { - if (this.store === null) { - throw new Error('Datastore needs to be opened.') - } try { - await this.store.put(this.location, val, key.toBuffer()) + await this.getStore('readwrite').put(val, key.toBuffer()) } catch (err) { throw Errors.dbWriteFailedError(err) } } async get (key) { - if (this.store === null) { - throw new Error('Datastore needs to be opened.') - } let value try { - value = await this.store.get(this.location, key.toBuffer()) + value = await this.getStore().get(key.toBuffer()) } catch (err) { throw Errors.dbWriteFailedError(err) } @@ -126,25 +140,29 @@ class IdbDatastore extends Adapter { return typedarrayToBuffer(value) } + /** + * Check if a key exists in the datastore + * + * @param {Key} key + * @returns {boolean} + */ async has (key) { - if (this.store === null) { - throw new Error('Datastore needs to be opened.') - } + let value try { - await this.get(key) + value = await this.getStore().getKey(key.toBuffer()) } catch (err) { - if (err.code === 'ERR_NOT_FOUND') return false - throw err + throw Errors.dbWriteFailedError(err) + } + + if (!value) { + return false } return true } async delete (key) { - if (this.store === null) { - throw new Error('Datastore needs to be opened.') - } try { - await this.store.delete(this.location, key.toBuffer()) + await this.getStore('readwrite').delete(key.toBuffer()) } catch (err) { throw Errors.dbDeleteFailedError(err) } @@ -162,10 +180,10 @@ class IdbDatastore extends Adapter { dels.push(key.toBuffer()) }, commit: async () => { - if (this.store === null) { + if (this.db === null) { throw new Error('Datastore needs to be opened.') } - const tx = this.store.transaction(this.location, 'readwrite') + const tx = this.db.transaction(this.location, 'readwrite') const store = tx.store await Promise.all(puts.map(p => store.put(p[1], p[0]))) await Promise.all(dels.map(p => store.delete(p))) @@ -175,10 +193,10 @@ class IdbDatastore extends Adapter { } query (q) { - if (this.store === null) { + if (this.db === null) { throw new Error('Datastore needs to be opened.') } - let it = queryIt(q, this.store, this.location) + let it = queryIt(q, this) if (Array.isArray(q.filters)) { it = q.filters.reduce((it, f) => filter(it, f), it) @@ -192,11 +210,11 @@ class IdbDatastore extends Adapter { } close () { - if (this.store === null) { + if (this.db === null) { throw new Error('Datastore needs to be opened.') } - this.store.close() - this.store = null + this.db.close() + this.db = null } destroy () { From 3797cf7a821c57c9497dd4ce1dc4823ce6c0bdc2 Mon Sep 17 00:00:00 2001 From: Hugo Dias Date: Tue, 16 Jun 2020 12:26:41 +0100 Subject: [PATCH 3/6] fix: tests name --- test/index.spec.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/index.spec.js b/test/index.spec.js index 8b88892..6016b48 100644 --- a/test/index.spec.js +++ b/test/index.spec.js @@ -6,10 +6,11 @@ const { Key } = require('interface-datastore') const { isNode } = require('ipfs-utils/src/env') const IDBStore = require('../src') -describe('LevelDatastore', function () { +describe('IndexedDB Datastore', function () { if (isNode) { return } + describe('interface-datastore (idb)', () => { const store = new IDBStore('hello') require('interface-datastore/src/tests')({ From 1b9eac850c3d84cef01b11f4c3db00e6f1f9c338 Mon Sep 17 00:00:00 2001 From: Hugo Dias Date: Tue, 16 Jun 2020 12:26:54 +0100 Subject: [PATCH 4/6] fix: use new tests --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 55da5e9..f714074 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ "dependencies": { "buffer": "^5.5.0", "idb": "^5.0.2", - "interface-datastore": "^1.0.2" + "interface-datastore": "ipfs/interface-datastore#test/add-tests-for-mutating-datastore-during-query" }, "devDependencies": { "aegir": "^22.0.0", From 0304a9a45094b8ef605020cd7dd2c543ffeab9fd Mon Sep 17 00:00:00 2001 From: Hugo Dias Date: Tue, 16 Jun 2020 12:30:57 +0100 Subject: [PATCH 5/6] fix: update deps and fix size --- .aegir.js | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.aegir.js b/.aegir.js index 9e9b7f6..6fc0ec6 100644 --- a/.aegir.js +++ b/.aegir.js @@ -1,3 +1,3 @@ module.exports = { - bundlesize: { maxSize: '12.1kB' } + bundlesize: { maxSize: '13kB' } } diff --git a/package.json b/package.json index f714074..544b788 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "interface-datastore": "ipfs/interface-datastore#test/add-tests-for-mutating-datastore-during-query" }, "devDependencies": { - "aegir": "^22.0.0", + "aegir": "^23.0.0", "chai": "^4.2.0", "datastore-core": "^1.1.0", "datastore-level": "^1.1.0", From 2a71763ce41f76d7e8edc8a421741ee2b9c33455 Mon Sep 17 00:00:00 2001 From: Hugo Dias Date: Tue, 16 Jun 2020 13:06:38 +0100 Subject: [PATCH 6/6] fix: add concurrency test --- test/index.spec.js | 68 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/test/index.spec.js b/test/index.spec.js index 6016b48..18d761f 100644 --- a/test/index.spec.js +++ b/test/index.spec.js @@ -1,6 +1,7 @@ /* eslint-env mocha */ 'use strict' +const { expect } = require('aegir/utils/chai') const { MountDatastore } = require('datastore-core') const { Key } = require('interface-datastore') const { isNode } = require('ipfs-utils/src/env') @@ -52,4 +53,71 @@ describe('IndexedDB Datastore', function () { } }) }) + + describe('concurrency', () => { + let store + + before(async () => { + store = new IDBStore('hello') + await store.open() + }) + + it('should not explode under unreasonable load', function (done) { + this.timeout(10000) + + const updater = setInterval(async () => { + try { + const key = new Key('/a-' + Date.now()) + + await store.put(key, Buffer.from([0, 1, 2, 3])) + await store.has(key) + await store.get(key) + } catch (err) { + clearInterval(updater) + clearInterval(mutatorQuery) + clearInterval(readOnlyQuery) + done(err) + } + }, 0) + + const mutatorQuery = setInterval(async () => { + try { + for await (const { key } of store.query({})) { + await store.get(key) + + const otherKey = new Key('/b-' + Date.now()) + const otherValue = Buffer.from([0, 1, 2, 3]) + await store.put(otherKey, otherValue) + const res = await store.get(otherKey) + expect(res).to.deep.equal(otherValue) + } + } catch (err) { + clearInterval(updater) + clearInterval(mutatorQuery) + clearInterval(readOnlyQuery) + done(err) + } + }, 0) + + const readOnlyQuery = setInterval(async () => { + try { + for await (const { key } of store.query({})) { + await store.has(key) + } + } catch (err) { + clearInterval(updater) + clearInterval(mutatorQuery) + clearInterval(readOnlyQuery) + done(err) + } + }, 0) + + setTimeout(() => { + clearInterval(updater) + clearInterval(mutatorQuery) + clearInterval(readOnlyQuery) + done() + }, 5000) + }) + }) })