From 66bba366eb145f470b1c40031e47a2d314809524 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 28 Jul 2020 14:47:45 +0100 Subject: [PATCH 1/2] fix: remove node buffers Updates to the latest interface-datastore that swaps out Buffers for Uint8Arrays. Depends on: - [ ] https://github.com/ipfs/interface-datastore/pull/43 BREAKING CHANGE: no longer uses node Buffers, only Uint8Arrays --- package.json | 6 +++--- src/shard.js | 3 ++- src/sharding.js | 6 +++--- src/utils.js | 7 +++++++ test/keytransform.spec.js | 3 +-- test/mount.spec.js | 16 ++++++++-------- test/namespace.spec.js | 4 ++-- test/sharding.spec.js | 10 +++++----- test/tiered.spec.js | 8 ++++---- 9 files changed, 35 insertions(+), 28 deletions(-) create mode 100644 src/utils.js diff --git a/package.json b/package.json index 5e45689..826e0ff 100644 --- a/package.json +++ b/package.json @@ -38,15 +38,15 @@ }, "homepage": "https://github.com/ipfs/js-datastore-core#readme", "devDependencies": { - "aegir": "^22.0.0", + "aegir": "^25.0.0", "async-iterator-all": "^1.0.0", "chai": "^4.2.0", "dirty-chai": "^2.0.1" }, "dependencies": { - "buffer": "^5.5.0", "debug": "^4.1.1", - "interface-datastore": "^1.0.2" + "interface-datastore": "ipfs/interface-datastore#fix/remove-node-buffer", + "ipfs-utils": "^2.3.1" }, "engines": { "node": ">=6.0.0", diff --git a/src/shard.js b/src/shard.js index 82027c1..a62dd95 100644 --- a/src/shard.js +++ b/src/shard.js @@ -2,6 +2,7 @@ 'use strict' const Key = require('interface-datastore').Key +const { utf8Decoder } = require('../src/utils') const readme = require('./shard-readme') @@ -125,7 +126,7 @@ exports.readShardFun = async (path /* : string */, store) /* : Promise const key = new Key(path).child(new Key(SHARDING_FN)) const get = typeof store.getRaw === 'function' ? store.getRaw.bind(store) : store.get.bind(store) const res = await get(key) - return parseShardFun((res || '').toString().trim()) + return parseShardFun(utf8Decoder.decode(res || '').trim()) } exports.readme = readme diff --git a/src/sharding.js b/src/sharding.js index 365235d..87c4d3d 100644 --- a/src/sharding.js +++ b/src/sharding.js @@ -1,9 +1,9 @@ 'use strict' -const { Buffer } = require('buffer') const { Adapter, Key } = require('interface-datastore') const sh = require('./shard') const KeytransformStore = require('./keytransform') +const { utf8Encoder } = require('../src/utils') const shardKey = new Key(sh.SHARDING_FN) const shardReadmeKey = new Key(sh.README_FN) @@ -65,8 +65,8 @@ class ShardingDatastore extends Adapter { const exists = await store.has(shardKey) if (!exists) { const put = typeof store.putRaw === 'function' ? store.putRaw.bind(store) : store.put.bind(store) - return Promise.all([put(shardKey, Buffer.from(shard.toString() + '\n')), - put(shardReadmeKey, Buffer.from(sh.readme))]) + return Promise.all([put(shardKey, utf8Encoder.encode(shard.toString() + '\n')), + put(shardReadmeKey, utf8Encoder.encode(sh.readme))]) } const diskShard = await sh.readShardFun('/', store) diff --git a/src/utils.js b/src/utils.js new file mode 100644 index 0000000..15e1356 --- /dev/null +++ b/src/utils.js @@ -0,0 +1,7 @@ +'use strict' + +const TextEncoder = require('ipfs-utils/src/text-encoder') +const TextDecoder = require('ipfs-utils/src/text-decoder') + +module.exports.utf8Encoder = new TextEncoder('utf8') +module.exports.utf8Decoder = new TextDecoder('utf8') diff --git a/test/keytransform.spec.js b/test/keytransform.spec.js index f058c1a..f75ec64 100644 --- a/test/keytransform.spec.js +++ b/test/keytransform.spec.js @@ -1,7 +1,6 @@ /* eslint-env mocha */ 'use strict' -const { Buffer } = require('buffer') const chai = require('chai') chai.use(require('dirty-chai')) const expect = chai.expect @@ -37,7 +36,7 @@ describe('KeyTransformDatastore', () => { 'foo/bar/bazb', 'foo/bar/baz/barb' ].map((s) => new Key(s)) - await Promise.all(keys.map((key) => kStore.put(key, Buffer.from(key.toString())))) + await Promise.all(keys.map((key) => kStore.put(key, key.toBuffer()))) const kResults = Promise.all(keys.map((key) => kStore.get(key))) const mResults = Promise.all(keys.map((key) => mStore.get(new Key('abc').child(key)))) const results = await Promise.all([kResults, mResults]) diff --git a/test/mount.spec.js b/test/mount.spec.js index a948610..eb24a57 100644 --- a/test/mount.spec.js +++ b/test/mount.spec.js @@ -2,12 +2,12 @@ /* eslint max-nested-callbacks: ["error", 8] */ 'use strict' -const { Buffer } = require('buffer') const chai = require('chai') chai.use(require('dirty-chai')) const assert = chai.assert const expect = chai.expect const all = require('async-iterator-all') +const { utf8Encoder } = require('../src/utils') const Key = require('interface-datastore').Key const MemoryStore = require('interface-datastore').MemoryDatastore @@ -18,7 +18,7 @@ describe('MountStore', () => { it('put - no mount', async () => { const m = new MountStore([]) try { - await m.put(new Key('hello'), Buffer.from('foo')) + await m.put(new Key('hello'), utf8Encoder.encode('foo')) assert(false, 'Failed to throw error on no mount') } catch (err) { expect(err).to.be.an('Error') @@ -31,7 +31,7 @@ describe('MountStore', () => { prefix: new Key('cool') }]) try { - await m.put(new Key('/fail/hello'), Buffer.from('foo')) + await m.put(new Key('/fail/hello'), utf8Encoder.encode('foo')) assert(false, 'Failed to throw error on wrong mount') } catch (err) { expect(err).to.be.an('Error') @@ -45,7 +45,7 @@ describe('MountStore', () => { prefix: new Key('cool') }]) - const val = Buffer.from('hello') + const val = utf8Encoder.encode('hello') await m.put(new Key('/cool/hello'), val) const res = await mds.get(new Key('/hello')) expect(res).to.eql(val) @@ -58,7 +58,7 @@ describe('MountStore', () => { prefix: new Key('cool') }]) - const val = Buffer.from('hello') + const val = utf8Encoder.encode('hello') await mds.put(new Key('/hello'), val) const res = await m.get(new Key('/cool/hello')) expect(res).to.eql(val) @@ -71,7 +71,7 @@ describe('MountStore', () => { prefix: new Key('cool') }]) - const val = Buffer.from('hello') + const val = utf8Encoder.encode('hello') await mds.put(new Key('/hello'), val) const exists = await m.has(new Key('/cool/hello')) expect(exists).to.eql(true) @@ -84,7 +84,7 @@ describe('MountStore', () => { prefix: new Key('cool') }]) - const val = Buffer.from('hello') + const val = utf8Encoder.encode('hello') await m.put(new Key('/cool/hello'), val) await m.delete(new Key('/cool/hello')) let exists = await m.has(new Key('/cool/hello')) @@ -100,7 +100,7 @@ describe('MountStore', () => { prefix: new Key('cool') }]) - const val = Buffer.from('hello') + const val = utf8Encoder.encode('hello') await m.put(new Key('/cool/hello'), val) const res = await all(m.query({ prefix: '/cool' })) expect(res).to.eql([{ key: new Key('/cool/hello'), value: val }]) diff --git a/test/namespace.spec.js b/test/namespace.spec.js index 7246889..5c4782d 100644 --- a/test/namespace.spec.js +++ b/test/namespace.spec.js @@ -1,7 +1,6 @@ /* eslint-env mocha */ 'use strict' -const { Buffer } = require('buffer') const chai = require('chai') chai.use(require('dirty-chai')) const expect = chai.expect @@ -11,6 +10,7 @@ const MemoryStore = require('interface-datastore').MemoryDatastore const NamespaceStore = require('../src/').NamespaceDatastore const all = require('async-iterator-all') +const { utf8Encoder } = require('../src/utils') describe('KeyTransformDatastore', () => { const prefixes = [ @@ -30,7 +30,7 @@ describe('KeyTransformDatastore', () => { 'foo/bar/baz/barb' ].map((s) => new Key(s)) - await Promise.all(keys.map(key => store.put(key, Buffer.from(key.toString())))) + await Promise.all(keys.map(key => store.put(key, utf8Encoder.encode(key.toString())))) const nResults = Promise.all(keys.map((key) => store.get(key))) const mResults = Promise.all(keys.map((key) => mStore.get(new Key(prefix).child(key)))) const results = await Promise.all([nResults, mResults]) diff --git a/test/sharding.spec.js b/test/sharding.spec.js index ad8ca9f..7a481c1 100644 --- a/test/sharding.spec.js +++ b/test/sharding.spec.js @@ -1,7 +1,6 @@ /* eslint-env mocha */ 'use strict' -const { Buffer } = require('buffer') const chai = require('chai') chai.use(require('dirty-chai')) const expect = chai.expect @@ -12,6 +11,7 @@ const MemoryStore = require('interface-datastore').MemoryDatastore const ShardingStore = require('../src').ShardingDatastore const sh = require('../src').shard +const { utf8Encoder, utf8Decoder } = require('../src/utils') describe('ShardingStore', () => { it('create', async () => { @@ -19,8 +19,8 @@ describe('ShardingStore', () => { const shard = new sh.NextToLast(2) await ShardingStore.create(ms, shard) const res = await Promise.all([ms.get(new Key(sh.SHARDING_FN)), ms.get(new Key(sh.README_FN))]) - expect(res[0].toString()).to.eql(shard.toString() + '\n') - expect(res[1].toString()).to.eql(sh.readme) + expect(utf8Decoder.decode(res[0])).to.eql(shard.toString() + '\n') + expect(utf8Decoder.decode(res[1])).to.eql(sh.readme) }) it('open - empty', async () => { @@ -47,9 +47,9 @@ describe('ShardingStore', () => { const store = await ShardingStore.createOrOpen(ms, shard) expect(store).to.exist() await ShardingStore.createOrOpen(ms, shard) - await store.put(new Key('hello'), Buffer.from('test')) + await store.put(new Key('hello'), utf8Encoder.encode('test')) const res = await ms.get(new Key('ll').child(new Key('hello'))) - expect(res).to.eql(Buffer.from('test')) + expect(res).to.eql(utf8Encoder.encode('test')) }) describe('interface-datastore', () => { diff --git a/test/tiered.spec.js b/test/tiered.spec.js index e8f2a96..fa308d2 100644 --- a/test/tiered.spec.js +++ b/test/tiered.spec.js @@ -1,7 +1,6 @@ /* eslint-env mocha */ 'use strict' -const { Buffer } = require('buffer') const chai = require('chai') chai.use(require('dirty-chai')) const expect = chai.expect @@ -10,6 +9,7 @@ const Key = require('interface-datastore').Key const MemoryStore = require('interface-datastore').MemoryDatastore const TieredStore = require('../src').TieredDatastore +const { utf8Encoder } = require('../src/utils') describe('Tiered', () => { describe('all stores', () => { @@ -23,7 +23,7 @@ describe('Tiered', () => { it('put', async () => { const k = new Key('hello') - const v = Buffer.from('world') + const v = utf8Encoder.encode('world') await store.put(k, v) const res = await Promise.all([ms[0].get(k), ms[1].get(k)]) res.forEach((val) => { @@ -33,7 +33,7 @@ describe('Tiered', () => { it('get and has, where available', async () => { const k = new Key('hello') - const v = Buffer.from('world') + const v = utf8Encoder.encode('world') await ms[1].put(k, v) const val = await store.get(k) expect(val).to.be.eql(v) @@ -47,7 +47,7 @@ describe('Tiered', () => { it('has and delete', async () => { const k = new Key('hello') - const v = Buffer.from('world') + const v = utf8Encoder.encode('world') await store.put(k, v) let res = await Promise.all([ms[0].has(k), ms[1].has(k)]) expect(res).to.be.eql([true, true]) From 8a3ce29d3b4029a57a2b1ce0437c93aa90fb4c16 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 29 Jul 2020 13:36:55 +0100 Subject: [PATCH 2/2] chore: remove gh dep version --- package.json | 2 +- test/keytransform.spec.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 826e0ff..32106c7 100644 --- a/package.json +++ b/package.json @@ -45,7 +45,7 @@ }, "dependencies": { "debug": "^4.1.1", - "interface-datastore": "ipfs/interface-datastore#fix/remove-node-buffer", + "interface-datastore": "^2.0.0", "ipfs-utils": "^2.3.1" }, "engines": { diff --git a/test/keytransform.spec.js b/test/keytransform.spec.js index f75ec64..dba5e81 100644 --- a/test/keytransform.spec.js +++ b/test/keytransform.spec.js @@ -36,7 +36,7 @@ describe('KeyTransformDatastore', () => { 'foo/bar/bazb', 'foo/bar/baz/barb' ].map((s) => new Key(s)) - await Promise.all(keys.map((key) => kStore.put(key, key.toBuffer()))) + await Promise.all(keys.map((key) => kStore.put(key, key.uint8Array()))) const kResults = Promise.all(keys.map((key) => kStore.get(key))) const mResults = Promise.all(keys.map((key) => mStore.get(new Key('abc').child(key)))) const results = await Promise.all([kResults, mResults])