diff --git a/package.json b/package.json index d57695f..244b835 100644 --- a/package.json +++ b/package.json @@ -196,7 +196,7 @@ "dependencies": { "@libp2p/logger": "^2.0.0", "err-code": "^3.0.1", - "interface-store": "^4.0.0", + "interface-store": "^5.0.1", "it-all": "^2.0.0", "it-drain": "^2.0.0", "it-filter": "^2.0.0", @@ -210,7 +210,7 @@ }, "devDependencies": { "aegir": "^38.1.7", - "interface-datastore": "^8.0.0", - "interface-datastore-tests": "^4.0.0" + "interface-datastore": "^8.1.2", + "interface-datastore-tests": "^5.0.0" } } diff --git a/src/base.ts b/src/base.ts index 132443b..821112e 100644 --- a/src/base.ts +++ b/src/base.ts @@ -2,48 +2,43 @@ import sort from 'it-sort' import drain from 'it-drain' import filter from 'it-filter' import take from 'it-take' -import type { Batch, Datastore, Key, KeyQuery, Options, Pair, Query } from 'interface-datastore' -import type { AwaitIterable } from 'interface-store' +import type { Batch, Datastore, Key, KeyQuery, Pair, Query } from 'interface-datastore' +import type { AbortOptions, Await, AwaitIterable } from 'interface-store' export class BaseDatastore implements Datastore { - async open (): Promise { - - } - - async close (): Promise { - + put (key: Key, val: Uint8Array, options?: AbortOptions): Await { + return Promise.reject(new Error('.put is not implemented')) } - async put (key: Key, val: Uint8Array, options?: Options): Promise { - await Promise.reject(new Error('.put is not implemented')) + get (key: Key, options?: AbortOptions): Await { + return Promise.reject(new Error('.get is not implemented')) } - async get (key: Key, options?: Options): Promise { - return await Promise.reject(new Error('.get is not implemented')) + has (key: Key, options?: AbortOptions): Await { + return Promise.reject(new Error('.has is not implemented')) } - async has (key: Key, options?: Options): Promise { - return await Promise.reject(new Error('.has is not implemented')) + delete (key: Key, options?: AbortOptions): Await { + return Promise.reject(new Error('.delete is not implemented')) } - async delete (key: Key, options?: Options): Promise { - await Promise.reject(new Error('.delete is not implemented')) - } - - async * putMany (source: AwaitIterable, options: Options = {}): AsyncIterable { + async * putMany (source: AwaitIterable, options: AbortOptions = {}): AwaitIterable { for await (const { key, value } of source) { await this.put(key, value, options) - yield { key, value } + yield key } } - async * getMany (source: AwaitIterable, options: Options = {}): AsyncIterable { + async * getMany (source: AwaitIterable, options: AbortOptions = {}): AwaitIterable { for await (const key of source) { - yield this.get(key, options) + yield { + key, + value: await this.get(key, options) + } } } - async * deleteMany (source: AwaitIterable, options: Options = {}): AsyncIterable { + async * deleteMany (source: AwaitIterable, options: AbortOptions = {}): AwaitIterable { for await (const key of source) { await this.delete(key, options) yield key @@ -75,7 +70,7 @@ export class BaseDatastore implements Datastore { * Extending classes should override `query` or implement this method */ // eslint-disable-next-line require-yield - async * _all (q: Query, options?: Options): AsyncIterable { + async * _all (q: Query, options?: AbortOptions): AwaitIterable { throw new Error('._all is not implemented') } @@ -83,11 +78,11 @@ export class BaseDatastore implements Datastore { * Extending classes should override `queryKeys` or implement this method */ // eslint-disable-next-line require-yield - async * _allKeys (q: KeyQuery, options?: Options): AsyncIterable { + async * _allKeys (q: KeyQuery, options?: AbortOptions): AwaitIterable { throw new Error('._allKeys is not implemented') } - query (q: Query, options?: Options): AsyncIterable { + query (q: Query, options?: AbortOptions): AwaitIterable { let it = this._all(q, options) if (q.prefix != null) { @@ -116,7 +111,7 @@ export class BaseDatastore implements Datastore { return it } - queryKeys (q: KeyQuery, options?: Options): AsyncIterable { + queryKeys (q: KeyQuery, options?: AbortOptions): AwaitIterable { let it = this._allKeys(q, options) if (q.prefix != null) { diff --git a/src/keytransform.ts b/src/keytransform.ts index efe967c..a9d2c9f 100644 --- a/src/keytransform.ts +++ b/src/keytransform.ts @@ -2,8 +2,8 @@ import { BaseDatastore } from './base.js' import map from 'it-map' import { pipe } from 'it-pipe' import type { KeyTransform } from './index.js' -import type { Batch, Datastore, Key, KeyQuery, Options, Pair, Query } from 'interface-datastore' -import type { AwaitIterable } from 'interface-store' +import type { Batch, Datastore, Key, KeyQuery, Pair, Query } from 'interface-datastore' +import type { AbortOptions, AwaitIterable } from 'interface-store' /** * A datastore shim, that wraps around a given datastore, changing @@ -21,23 +21,25 @@ export class KeyTransformDatastore extends BaseDatastore { this.transform = transform } - async put (key: Key, val: Uint8Array, options?: Options): Promise { + async put (key: Key, val: Uint8Array, options?: AbortOptions): Promise { await this.child.put(this.transform.convert(key), val, options) + + return key } - async get (key: Key, options?: Options): Promise { + async get (key: Key, options?: AbortOptions): Promise { return await this.child.get(this.transform.convert(key), options) } - async has (key: Key, options?: Options): Promise { + async has (key: Key, options?: AbortOptions): Promise { return await this.child.has(this.transform.convert(key), options) } - async delete (key: Key, options?: Options): Promise { + async delete (key: Key, options?: AbortOptions): Promise { await this.child.delete(this.transform.convert(key), options) } - async * putMany (source: AwaitIterable, options: Options = {}): AsyncIterable { + async * putMany (source: AwaitIterable, options: AbortOptions = {}): AsyncIterable { const transform = this.transform const child = this.child @@ -53,15 +55,12 @@ export class KeyTransformDatastore extends BaseDatastore { yield * child.putMany(source, options) }, async function * (source) { - yield * map(source, ({ key, value }) => ({ - key: transform.invert(key), - value - })) + yield * map(source, key => transform.invert(key)) } ) } - async * getMany (source: AwaitIterable, options: Options = {}): AsyncIterable { + async * getMany (source: AwaitIterable, options: AbortOptions = {}): AsyncIterable { const transform = this.transform const child = this.child @@ -72,11 +71,17 @@ export class KeyTransformDatastore extends BaseDatastore { }, async function * (source) { yield * child.getMany(source, options) + }, + async function * (source) { + yield * map(source, ({ key, value }) => ({ + key: transform.invert(key), + value + })) } ) } - async * deleteMany (source: AwaitIterable, options: Options = {}): AsyncIterable { + async * deleteMany (source: AwaitIterable, options: AbortOptions = {}): AsyncIterable { const transform = this.transform const child = this.child @@ -109,7 +114,7 @@ export class KeyTransformDatastore extends BaseDatastore { } } - query (q: Query, options?: Options): AsyncIterable { + query (q: Query, options?: AbortOptions): AsyncIterable { const query: Query = { ...q } @@ -143,7 +148,7 @@ export class KeyTransformDatastore extends BaseDatastore { }) } - queryKeys (q: KeyQuery, options?: Options): AsyncIterable { + queryKeys (q: KeyQuery, options?: AbortOptions): AsyncIterable { const query = { ...q } diff --git a/src/memory.ts b/src/memory.ts index 191a7d5..ad5cce4 100644 --- a/src/memory.ts +++ b/src/memory.ts @@ -2,6 +2,7 @@ import { BaseDatastore } from './base.js' import { Key } from 'interface-datastore/key' import * as Errors from './errors.js' import type { Pair } from 'interface-datastore' +import type { Await, AwaitIterable } from 'interface-store' export class MemoryDatastore extends BaseDatastore { private readonly data: Map @@ -12,11 +13,13 @@ export class MemoryDatastore extends BaseDatastore { this.data = new Map() } - async put (key: Key, val: Uint8Array): Promise { // eslint-disable-line require-await + put (key: Key, val: Uint8Array): Await { // eslint-disable-line require-await this.data.set(key.toString(), val) + + return key } - async get (key: Key): Promise { + get (key: Key): Await { const result = this.data.get(key.toString()) if (result == null) { @@ -26,21 +29,21 @@ export class MemoryDatastore extends BaseDatastore { return result } - async has (key: Key): Promise { // eslint-disable-line require-await + has (key: Key): Await { // eslint-disable-line require-await return this.data.has(key.toString()) } - async delete (key: Key): Promise { // eslint-disable-line require-await + delete (key: Key): Await { // eslint-disable-line require-await this.data.delete(key.toString()) } - async * _all (): AsyncIterable { + * _all (): AwaitIterable { for (const [key, value] of this.data.entries()) { yield { key: new Key(key), value } } } - async * _allKeys (): AsyncIterable { + * _allKeys (): AwaitIterable { for (const key of this.data.keys()) { yield new Key(key) } diff --git a/src/mount.ts b/src/mount.ts index 6e498a3..80e7b8c 100644 --- a/src/mount.ts +++ b/src/mount.ts @@ -5,7 +5,7 @@ import { BaseDatastore } from './base.js' import * as Errors from './errors.js' import sort from 'it-sort' import type { Batch, Datastore, Key, KeyQuery, Pair, Query } from 'interface-datastore' -import type { Options } from 'interface-store' +import type { AbortOptions } from 'interface-store' /** * A datastore that can combine multiple stores inside various @@ -34,20 +34,22 @@ export class MountDatastore extends BaseDatastore { } } - async put (key: Key, value: Uint8Array, options?: Options): Promise { + async put (key: Key, value: Uint8Array, options?: AbortOptions): Promise { const match = this._lookup(key) if (match == null) { throw Errors.dbWriteFailedError(new Error('No datastore mounted for this key')) } await match.datastore.put(key, value, options) + + return key } /** * @param {Key} key * @param {Options} [options] */ - async get (key: Key, options: Options = {}): Promise { + async get (key: Key, options: AbortOptions = {}): Promise { const match = this._lookup(key) if (match == null) { throw Errors.notFoundError(new Error('No datastore mounted for this key')) @@ -55,7 +57,7 @@ export class MountDatastore extends BaseDatastore { return await match.datastore.get(key, options) } - async has (key: Key, options?: Options): Promise { + async has (key: Key, options?: AbortOptions): Promise { const match = this._lookup(key) if (match == null) { return await Promise.resolve(false) @@ -63,7 +65,7 @@ export class MountDatastore extends BaseDatastore { return await match.datastore.has(key, options) } - async delete (key: Key, options?: Options): Promise { + async delete (key: Key, options?: AbortOptions): Promise { const match = this._lookup(key) if (match == null) { throw Errors.dbDeleteFailedError(new Error('No datastore mounted for this key')) @@ -106,7 +108,7 @@ export class MountDatastore extends BaseDatastore { } } - query (q: Query, options?: Options): AsyncIterable { + query (q: Query, options?: AbortOptions): AsyncIterable { const qs = this.mounts.map(m => { return m.datastore.query({ prefix: q.prefix, @@ -127,7 +129,7 @@ export class MountDatastore extends BaseDatastore { return it } - queryKeys (q: KeyQuery, options?: Options): AsyncIterable { + queryKeys (q: KeyQuery, options?: AbortOptions): AsyncIterable { const qs = this.mounts.map(m => { return m.datastore.queryKeys({ prefix: q.prefix, diff --git a/src/sharding.ts b/src/sharding.ts index 9152175..06e191d 100644 --- a/src/sharding.ts +++ b/src/sharding.ts @@ -1,4 +1,4 @@ -import { Batch, Key, KeyQuery, KeyQueryFilter, Options, Pair, Query, QueryFilter } from 'interface-datastore' +import { Batch, Key, KeyQuery, KeyQueryFilter, Pair, Query, QueryFilter } from 'interface-datastore' import { readShardFun, SHARDING_FN @@ -8,7 +8,7 @@ import { KeyTransformDatastore } from './keytransform.js' import * as Errors from './errors.js' import type { Shard } from './index.js' import type { Datastore } from 'interface-datastore' -import type { AwaitIterable } from 'interface-store' +import type { AbortOptions, AwaitIterable } from 'interface-store' const shardKey = new Key(SHARDING_FN) @@ -81,31 +81,33 @@ export class ShardingDatastore extends BaseDatastore { return diskShard } - async put (key: Key, val: Uint8Array, options?: Options): Promise { + async put (key: Key, val: Uint8Array, options?: AbortOptions): Promise { await this.child.put(key, val, options) + + return key } - async get (key: Key, options?: Options): Promise { + async get (key: Key, options?: AbortOptions): Promise { return await this.child.get(key, options) } - async has (key: Key, options?: Options): Promise { + async has (key: Key, options?: AbortOptions): Promise { return await this.child.has(key, options) } - async delete (key: Key, options?: Options): Promise { + async delete (key: Key, options?: AbortOptions): Promise { await this.child.delete(key, options) } - async * putMany (source: AwaitIterable, options: Options = {}): AsyncIterable { + async * putMany (source: AwaitIterable, options: AbortOptions = {}): AsyncIterable { yield * this.child.putMany(source, options) } - async * getMany (source: AwaitIterable, options: Options = {}): AsyncIterable { + async * getMany (source: AwaitIterable, options: AbortOptions = {}): AsyncIterable { yield * this.child.getMany(source, options) } - async * deleteMany (source: AwaitIterable, options: Options = {}): AsyncIterable { + async * deleteMany (source: AwaitIterable, options: AbortOptions = {}): AsyncIterable { yield * this.child.deleteMany(source, options) } @@ -113,7 +115,7 @@ export class ShardingDatastore extends BaseDatastore { return this.child.batch() } - query (q: Query, options?: Options): AsyncIterable { + query (q: Query, options?: AbortOptions): AsyncIterable { const omitShard: QueryFilter = ({ key }) => key.toString() !== shardKey.toString() const tq: Query = { @@ -126,7 +128,7 @@ export class ShardingDatastore extends BaseDatastore { return this.child.query(tq, options) } - queryKeys (q: KeyQuery, options?: Options): AsyncIterable { + queryKeys (q: KeyQuery, options?: AbortOptions): AsyncIterable { const omitShard: KeyQueryFilter = (key) => key.toString() !== shardKey.toString() const tq: KeyQuery = { diff --git a/src/tiered.ts b/src/tiered.ts index cbf27ca..c42084d 100644 --- a/src/tiered.ts +++ b/src/tiered.ts @@ -3,8 +3,8 @@ import * as Errors from './errors.js' import { logger } from '@libp2p/logger' import { pushable } from 'it-pushable' import drain from 'it-drain' -import type { Batch, Datastore, Key, KeyQuery, Options, Pair, Query } from 'interface-datastore' -import type { AwaitIterable } from 'interface-store' +import type { Batch, Datastore, Key, KeyQuery, Pair, Query } from 'interface-datastore' +import type { AbortOptions, AwaitIterable } from 'interface-store' const log = logger('datastore:core:tiered') @@ -24,15 +24,16 @@ export class TieredDatastore extends BaseDatastore { this.stores = stores.slice() } - async put (key: Key, value: Uint8Array, options?: Options): Promise { + async put (key: Key, value: Uint8Array, options?: AbortOptions): Promise { try { await Promise.all(this.stores.map(async store => { await store.put(key, value, options) })) + return key } catch (err: any) { throw Errors.dbWriteFailedError(err) } } - async get (key: Key, options?: Options): Promise { + async get (key: Key, options?: AbortOptions): Promise { for (const store of this.stores) { try { const res = await store.get(key, options) @@ -44,7 +45,7 @@ export class TieredDatastore extends BaseDatastore { throw Errors.notFoundError() } - async has (key: Key, options?: Options): Promise { + async has (key: Key, options?: AbortOptions): Promise { for (const s of this.stores) { if (await s.has(key, options)) { return true @@ -54,7 +55,7 @@ export class TieredDatastore extends BaseDatastore { return false } - async delete (key: Key, options?: Options): Promise { + async delete (key: Key, options?: AbortOptions): Promise { try { await Promise.all(this.stores.map(async store => { await store.delete(key, options) })) } catch (err: any) { @@ -62,7 +63,7 @@ export class TieredDatastore extends BaseDatastore { } } - async * putMany (source: AwaitIterable, options: Options = {}): AsyncIterable { + async * putMany (source: AwaitIterable, options: AbortOptions = {}): AsyncIterable { let error: Error | undefined const pushables = this.stores.map(store => { const source = pushable({ @@ -86,14 +87,14 @@ export class TieredDatastore extends BaseDatastore { pushables.forEach(p => p.push(pair)) - yield pair + yield pair.key } } finally { pushables.forEach(p => p.end()) } } - async * deleteMany (source: AwaitIterable, options: Options = {}): AsyncIterable { + async * deleteMany (source: AwaitIterable, options: AbortOptions = {}): AsyncIterable { let error: Error | undefined const pushables = this.stores.map(store => { const source = pushable({ @@ -142,11 +143,11 @@ export class TieredDatastore extends BaseDatastore { } } - query (q: Query, options?: Options): AsyncIterable { + query (q: Query, options?: AbortOptions): AwaitIterable { return this.stores[this.stores.length - 1].query(q, options) } - queryKeys (q: KeyQuery, options?: Options): AsyncIterable { + queryKeys (q: KeyQuery, options?: AbortOptions): AwaitIterable { return this.stores[this.stores.length - 1].queryKeys(q, options) } }