diff --git a/README.md b/README.md index a5d28e3..d22f2d1 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,11 @@ This package is inspired by the [go-ipfs repo migration tool](https://github.com - [Tests](#tests) - [Empty migrations](#empty-migrations) - [Migrations matrix](#migrations-matrix) + - [Migrations](#migrations) + - [7](#7) + - [8](#8) + - [9](#9) + - [10](#10) - [Developer](#developer) - [Module versioning notes](#module-versioning-notes) - [Contribute](#contribute) @@ -268,6 +273,24 @@ This will create an empty migration with the next version. | 8 | v0.48.0 | | 9 | v0.49.0 | +### Migrations + +#### 7 + +This is the initial version of the datastore, inherited from go-IPFS in an attempt to maintain cross-compatibility between the two implementations. + +#### 8 + +Blockstore keys are transformed into base32 representations of the multihash from the CID of the block. + +#### 9 + +Pins were migrated from a DAG to a Datastore - see [ipfs/js-ipfs#2771](https://github.com/ipfs/js-ipfs/pull/2771) + +#### 10 + +`level@6.x.x` upgrades the `level-js` dependency from `4.x.x` to `5.x.x`. This update requires a database migration to convert all string keys/values into buffers. Only runs in the browser, node is unaffected. See [Level/level-js#179](https://github.com/Level/level-js/pull/179) + ## Developer ### Module versioning notes diff --git a/migrations/index.js b/migrations/index.js index 11ceced..91b83df 100644 --- a/migrations/index.js +++ b/migrations/index.js @@ -16,5 +16,6 @@ module.exports = [ Object.assign({version: 6}, emptyMigration), Object.assign({version: 7}, emptyMigration), require('./migration-8'), - require('./migration-9') + require('./migration-9'), + require('./migration-10') ] diff --git a/migrations/migration-10/index.js b/migrations/migration-10/index.js new file mode 100644 index 0000000..3a96154 --- /dev/null +++ b/migrations/migration-10/index.js @@ -0,0 +1,157 @@ +'use strict' + +const { + createStore, + findLevelJs +} = require('../../src/utils') +const { Key } = require('interface-datastore') +const fromString = require('uint8arrays/from-string') +const toString = require('uint8arrays/to-string') + +async function keysToBinary (name, store, onProgress = () => {}) { + let db = findLevelJs(store) + + // only interested in level-js + if (!db) { + onProgress(`${name} did not need an upgrade`) + + return + } + + onProgress(`Upgrading ${name}`) + + await withEach(db, (key, value) => { + return [ + { type: 'del', key: key }, + { type: 'put', key: fromString(key), value: value } + ] + }) +} + +async function keysToStrings (name, store, onProgress = () => {}) { + let db = findLevelJs(store) + + // only interested in level-js + if (!db) { + onProgress(`${name} did not need a downgrade`) + + return + } + + onProgress(`Downgrading ${name}`) + + await withEach(db, (key, value) => { + return [ + { type: 'del', key: key }, + { type: 'put', key: toString(key), value: value } + ] + }) +} + +async function process (repoPath, repoOptions, onProgress, fn) { + const datastores = Object.keys(repoOptions.storageBackends) + .filter(key => repoOptions.storageBackends[key].name === 'LevelDatastore') + .map(name => ({ + name, + store: createStore(repoPath, name, repoOptions) + })) + + onProgress(0, `Migrating ${datastores.length} dbs`) + let migrated = 0 + + for (const { name, store } of datastores) { + await store.open() + + try { + await fn(name, store, (message) => { + onProgress(parseInt((migrated / datastores.length) * 100), message) + }) + } finally { + migrated++ + store.close() + } + } + + onProgress(100, `Migrated ${datastores.length} dbs`) +} + +module.exports = { + version: 10, + description: 'Migrates datastore-level keys to binary', + migrate: (repoPath, repoOptions, onProgress = () => {}) => { + return process(repoPath, repoOptions, onProgress, keysToBinary) + }, + revert: (repoPath, repoOptions, onProgress = () => {}) => { + return process(repoPath, repoOptions, onProgress, keysToStrings) + } +} + +/** + * @typedef {Uint8Array|string} Key + * @typedef {Uint8Array} Value + * @typedef {{ type: 'del', key: Key } | { type: 'put', key: Key, value: Value }} Operation + * + * Uses the upgrade strategy from level-js@5.x.x - note we can't call the `.upgrade` command + * directly because it will be removed in level-js@6.x.x and we can't guarantee users will + * have migrated by then - e.g. they may jump from level-js@4.x.x straight to level-js@6.x.x + * so we have to duplicate the code here. + * + * @param {import('interface-datastore').Datastore} db + * @param {function (Key, Value): Operation[]} fn + */ +function withEach (db, fn) { + function batch (operations, next) { + const store = db.store('readwrite') + const transaction = store.transaction + let index = 0 + let error + + transaction.onabort = () => next(error || transaction.error || new Error('aborted by user')) + transaction.oncomplete = () => next() + + function loop () { + var op = operations[index++] + var key = op.key + + try { + var req = op.type === 'del' ? store.delete(key) : store.put(op.value, key) + } catch (err) { + error = err + transaction.abort() + return + } + + if (index < operations.length) { + req.onsuccess = loop + } + } + + loop() + } + + return new Promise((resolve, reject) => { + const it = db.iterator() + // raw keys and values only + it._deserializeKey = it._deserializeValue = (data) => data + next() + + function next () { + it.next((err, key, value) => { + if (err || key === undefined) { + it.end((err2) => { + if (err2) { + reject(err2) + return + } + + resolve() + }) + + return + } + + batch(fn(key, value), next) + }) + } + }) +} diff --git a/migrations/migration-8/index.js b/migrations/migration-8/index.js index 3512a95..f624529 100644 --- a/migrations/migration-8/index.js +++ b/migrations/migration-8/index.js @@ -75,10 +75,10 @@ async function process (repoPath, repoOptions, onProgress, keyFunction) { module.exports = { version: 8, description: 'Transforms key names into base32 encoding and converts Block store to use bare multihashes encoded as base32', - migrate: (repoPath, repoOptions, onProgress) => { + migrate: (repoPath, repoOptions, onProgress = () => {}) => { return process(repoPath, repoOptions, onProgress, keyToMultihash) }, - revert: (repoPath, repoOptions, onProgress) => { + revert: (repoPath, repoOptions, onProgress = () => {}) => { return process(repoPath, repoOptions, onProgress, keyToCid) } } diff --git a/migrations/migration-9/index.js b/migrations/migration-9/index.js index 243e64f..2d8a9c8 100644 --- a/migrations/migration-9/index.js +++ b/migrations/migration-9/index.js @@ -135,10 +135,10 @@ async function process (repoPath, repoOptions, onProgress, fn) { module.exports = { version: 9, description: 'Migrates pins to datastore', - migrate: (repoPath, repoOptions, onProgress) => { + migrate: (repoPath, repoOptions, onProgress = () => {}) => { return process(repoPath, repoOptions, onProgress, pinsToDatastore) }, - revert: (repoPath, repoOptions, onProgress) => { + revert: (repoPath, repoOptions, onProgress = () => {}) => { return process(repoPath, repoOptions, onProgress, pinsToDAG) } } diff --git a/package.json b/package.json index cde37fd..e1c87f8 100644 --- a/package.json +++ b/package.json @@ -64,6 +64,8 @@ "datastore-level": "^3.0.0", "it-all": "^1.0.2", "just-safe-set": "^2.1.0", + "level-5": "npm:level@^5.0.0", + "level-6": "npm:level@^6.0.0", "ncp": "^2.0.0", "rimraf": "^3.0.0", "sinon": "^9.0.2" diff --git a/src/index.js b/src/index.js index f6ecec3..8d9e459 100644 --- a/src/index.js +++ b/src/index.js @@ -121,7 +121,7 @@ async function migrate (path, repoOptions, toVersion, { ignoreLock = false, onPr await repoVersion.setVersion(path, toVersion || getLatestMigrationVersion(migrations), repoOptions) } - log('Repo successfully migrated ', toVersion !== undefined ? `to version ${toVersion}!` : 'to latest version!') + log('Repo successfully migrated', toVersion !== undefined ? `to version ${toVersion}!` : 'to latest version!') } finally { if (!isDryRun && !ignoreLock) { await lock.close() diff --git a/src/repo/version.js b/src/repo/version.js index ec3d4ad..1aeabca 100644 --- a/src/repo/version.js +++ b/src/repo/version.js @@ -4,6 +4,7 @@ const repoInit = require('./init') const { MissingRepoOptionsError, NotInitializedRepoError } = require('../errors') const { VERSION_KEY, createStore } = require('../utils') const uint8ArrayFromString = require('uint8arrays/from-string') +const uint8ArrayToString = require('uint8arrays/to-string') exports.getVersion = getVersion @@ -28,7 +29,14 @@ async function getVersion (path, repoOptions) { const store = createStore(path, 'root', repoOptions) await store.open() - const version = parseInt(await store.get(VERSION_KEY)) + let version = await store.get(VERSION_KEY) + + if (version instanceof Uint8Array) { + version = uint8ArrayToString(version) + } + + version = parseInt(version) + await store.close() return version diff --git a/src/utils.js b/src/utils.js index bc24597..d0748b1 100644 --- a/src/utils.js +++ b/src/utils.js @@ -1,11 +1,19 @@ 'use strict' -const Key = require('interface-datastore').Key +const { + Key, + Errors +} = require('interface-datastore') const core = require('datastore-core') const ShardingStore = core.ShardingDatastore -exports.CONFIG_KEY = new Key('/config') -exports.VERSION_KEY = new Key('/version') +/** + * @typedef {import('interface-datastore').Key} Key + * @typedef {import('interface-datastore').Datastore} Datastore + */ + +const CONFIG_KEY = new Key('/config') +const VERSION_KEY = new Key('/version') function getDatastoreAndOptions (name, options) { if (!options || !options.storageBackends) { @@ -30,6 +38,94 @@ function getDatastoreAndOptions (name, options) { } } +/** + * Level dbs wrap level dbs that wrap level dbs. Find a level-js + * instance in the chain if one exists. + * + * @param {Datastore} store + */ +function findLevelJs (store) { + let db = store + + while (db.db || db.child) { + db = db.db || db.child + + // `Level` is only present in the browser, in node it is LevelDOWN + if (db.type === 'level-js' || db.constructor.name === 'Level') { + return db + } + } +} + +/** + * @param {Key} key + * @param {function (Key): Promise} has + * @param {Datastore} store + */ +async function hasWithFallback (key, has, store) { + const result = await has(key) + + if (result) { + return result + } + + // Newer versions of level.js changed the key type from Uint8Array|string + // to Uint8Array so fall back to trying Uint8Arrays if we are using level.js + // and the string version of the key did not work + const levelJs = findLevelJs(store) + + if (!levelJs) { + return false + } + + return new Promise((resolve, reject) => { + // drop down to IndexDB API, otherwise level-js will monkey around with the keys/values + const req = levelJs.store('readonly').get(key.toString()) + req.transaction.onabort = () => { + reject(req.transaction.error) + } + req.transaction.oncomplete = () => { + resolve(Boolean(req.result)) + } + }) +} + +/** + * @param {import('interface-datastore').Key} key + * @param {function (Key): Promise} get + * @param {function (Key): Promise} has + * @param {import('interface-datastore').Datastore} store + */ +async function getWithFallback (key, get, has, store) { + if (await has(key)) { + return get(key) + } + + // Newer versions of level.js changed the key type from Uint8Array|string + // to Uint8Array so fall back to trying Uint8Arrays if we are using level.js + // and the string version of the key did not work + const levelJs = findLevelJs(store) + + if (!levelJs) { + throw Errors.notFoundError() + } + + return new Promise((resolve, reject) => { + // drop down to IndexDB API, otherwise level-js will monkey around with the keys/values + const req = levelJs.store('readonly').get(key.toString()) + req.transaction.onabort = () => { + reject(req.transaction.error) + } + req.transaction.oncomplete = () => { + if (req.result) { + return resolve(req.result) + } + + reject(Errors.notFoundError()) + } + }) +} + function createStore (location, name, options) { const { StorageBackend, storageOptions } = getDatastoreAndOptions(name, options) @@ -43,14 +139,20 @@ function createStore (location, name, options) { store = new ShardingStore(store, new core.shard.NextToLast(2)) } + // necessary since level-js@5 cannot read keys from level-js@4 and earlier + const originalGet = store.get.bind(store) + const originalHas = store.has.bind(store) + store.get = (key) => getWithFallback(key, originalGet, originalHas, store) + store.has = (key) => hasWithFallback(key, originalHas, store) + return store } -function containsIrreversibleMigration (from, to, migrations) { - return migrations - .filter(migration => migration.version > from && migration.version <= to) - .some(migration => migration.revert === undefined) +module.exports = { + createStore, + hasWithFallback, + getWithFallback, + findLevelJs, + CONFIG_KEY, + VERSION_KEY } - -exports.createStore = createStore -exports.containsIrreversibleMigration = containsIrreversibleMigration diff --git a/test/browser.js b/test/browser.js index 2fd1640..f9207e4 100644 --- a/test/browser.js +++ b/test/browser.js @@ -1,16 +1,17 @@ /* eslint-env mocha */ 'use strict' +const DatastoreLevel = require('datastore-level') const { createRepo, createAndLoadRepo } = require('./fixtures/repo') const repoOptions = { lock: 'memory', storageBackends: { - root: require('datastore-level'), - blocks: require('datastore-level'), - keys: require('datastore-level'), - datastore: require('datastore-level'), - pins: require('datastore-level') + root: DatastoreLevel, + blocks: DatastoreLevel, + keys: DatastoreLevel, + datastore: DatastoreLevel, + pins: DatastoreLevel }, storageBackendOptions: { root: { @@ -51,7 +52,7 @@ describe('Browser specific tests', () => { }) describe('migrations tests', () => { - require('./migrations')(() => createRepo(repoOptions), repoCleanup, repoOptions) + require('./migrations')(() => createRepo(repoOptions), repoCleanup) }) describe('init tests', () => { diff --git a/test/migrations/index.js b/test/migrations/index.js index 50c3016..bdc4129 100644 --- a/test/migrations/index.js +++ b/test/migrations/index.js @@ -77,6 +77,7 @@ module.exports = (createRepo, repoCleanup) => { describe(name, () => { require('./migration-8-test')(createRepo, repoCleanup, options) require('./migration-9-test')(createRepo, repoCleanup, options) + require('./migration-10-test')(createRepo, repoCleanup, options) }) }) } diff --git a/test/migrations/migration-10-test.js b/test/migrations/migration-10-test.js new file mode 100644 index 0000000..b3340fc --- /dev/null +++ b/test/migrations/migration-10-test.js @@ -0,0 +1,125 @@ +/* eslint-env mocha */ +/* eslint-disable max-nested-callbacks */ +'use strict' + +const { expect } = require('aegir/utils/chai') + +const { createStore } = require('../../src/utils') +const migration = require('../../migrations/migration-10') +const Key = require('interface-datastore').Key +const fromString = require('uint8arrays/from-string') +const Level5 = require('level-5') +const Level6 = require('level-6') + +const keys = { + CIQCKN76QUQUGYCHIKGFE6V6P3GJ2W26YFFPQW6YXV7NFHH3QB2RI3I: 'hello', + CIQKKLBWAIBQZOIS5X7E32LQAL6236OUKZTMHPQSFIXPWXNZHQOV7JQ: fromString('derp') +} + +async function bootstrap (dir, backend, repoOptions) { + const store = createStore(dir, backend, repoOptions) + await store.open() + + for (const name of Object.keys(keys)) { + await store.put(new Key(name), keys[name]) + } + + await store.close() +} + +async function validate (dir, backend, repoOptions) { + const store = createStore(dir, backend, repoOptions) + + await store.open() + + for (const name of Object.keys(keys)) { + const key = new Key(`/${name}`) + + expect(await store.has(key)).to.be.true(`Could not read key ${name}`) + expect(store.get(key)).to.eventually.equal(keys[name], `Could not read value for key ${keys[name]}`) + } + + await store.close() +} + +function withLevel (repoOptions, levelImpl) { + const stores = Object.keys(repoOptions.storageBackends) + .filter(key => repoOptions.storageBackends[key].name === 'LevelDatastore') + + const output = { + ...repoOptions + } + + stores.forEach(store => { + // override version of level passed to datastore options + output.storageBackendOptions[store] = { + ...output.storageBackendOptions[store], + db: levelImpl + } + }) + + return output +} + +module.exports = (setup, cleanup, repoOptions) => { + describe('migration 10', function () { + this.timeout(240 * 1000) + let dir + + beforeEach(async () => { + dir = await setup() + }) + + afterEach(async () => { + await cleanup(dir) + }) + + describe('forwards', () => { + beforeEach(async () => { + for (const backend of Object.keys(repoOptions.storageBackends)) { + await bootstrap(dir, backend, withLevel(repoOptions, Level5)) + } + }) + + it('should migrate keys and values forward', async () => { + await migration.migrate(dir, withLevel(repoOptions, Level6), () => {}) + + for (const backend of Object.keys(repoOptions.storageBackends)) { + await validate(dir, backend, withLevel(repoOptions, Level6)) + } + }) + }) + + describe('backwards using level@6.x.x', () => { + beforeEach(async () => { + for (const backend of Object.keys(repoOptions.storageBackends)) { + await bootstrap(dir, backend, withLevel(repoOptions, Level6)) + } + }) + + it('should migrate keys and values backward', async () => { + await migration.revert(dir, withLevel(repoOptions, Level6), () => {}) + + for (const backend of Object.keys(repoOptions.storageBackends)) { + await validate(dir, backend, withLevel(repoOptions, Level5)) + } + }) + }) + + describe('backwards using level@5.x.x', () => { + beforeEach(async () => { + for (const backend of Object.keys(repoOptions.storageBackends)) { + await bootstrap(dir, backend, withLevel(repoOptions, Level6)) + } + }) + + it('should migrate keys and values backward', async () => { + await migration.revert(dir, withLevel(repoOptions, Level5), () => {}) + + for (const backend of Object.keys(repoOptions.storageBackends)) { + await validate(dir, backend, withLevel(repoOptions, Level5)) + } + }) + }) + }) +} diff --git a/test/node.js b/test/node.js index 7e97142..6c26abe 100644 --- a/test/node.js +++ b/test/node.js @@ -1,6 +1,8 @@ /* eslint-env mocha */ 'use strict' +const DatastoreFS = require('datastore-fs') +const DatastoreLevel = require('datastore-level') const promisify = require('util').promisify const asyncRimraf = promisify(require('rimraf')) const { createRepo, createAndLoadRepo } = require('./fixtures/repo') @@ -9,11 +11,11 @@ const os = require('os') const repoOptions = { lock: 'fs', storageBackends: { - root: require('datastore-fs'), - blocks: require('datastore-fs'), - keys: require('datastore-fs'), - datastore: require('datastore-level'), - pins: require('datastore-level') + root: DatastoreFS, + blocks: DatastoreFS, + keys: DatastoreFS, + datastore: DatastoreLevel, + pins: DatastoreLevel }, storageBackendOptions: { root: { @@ -50,7 +52,7 @@ describe('Node specific tests', () => { }) describe('migrations tests', () => { - require('./migrations')(() => createRepo(repoOptions, os.tmpdir()), repoCleanup, repoOptions) + require('./migrations')(() => createRepo(repoOptions, os.tmpdir()), repoCleanup) }) describe('init tests', () => {