Skip to content

Commit 332a794

Browse files
authored
feat: add 'extended' option to exporter (#437)
To list directories without resolving the root node of each directory entry, add an `extended` option to the exporter. Defaults to `true` to preserve backwards compatibility.
1 parent 93cb3d0 commit 332a794

File tree

6 files changed

+206
-12
lines changed

6 files changed

+206
-12
lines changed

packages/ipfs-unixfs-exporter/src/index.ts

Lines changed: 50 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ import type { PBNode } from '@ipld/dag-pb'
5757
import type { Bucket } from 'hamt-sharding'
5858
import type { Blockstore } from 'interface-blockstore'
5959
import type { UnixFS } from 'ipfs-unixfs'
60+
import type { AbortOptions } from 'it-pushable'
6061
import type { ProgressOptions, ProgressEvent } from 'progress-events'
6162

6263
export * from './errors.js'
@@ -136,6 +137,21 @@ export interface ExporterOptions extends ProgressOptions<ExporterProgressEvents>
136137
blockReadConcurrency?: number
137138
}
138139

140+
export interface BasicExporterOptions extends ExporterOptions {
141+
/**
142+
* When directory contents are listed, by default the root node of each entry
143+
* is fetched to decode the UnixFS metadata and know if the entry is a file or
144+
* a directory. This can result in fetching extra data which may not be
145+
* desirable, depending on your application.
146+
*
147+
* Pass false here to only return the CID and the name of the entry and not
148+
* any extended metadata.
149+
*
150+
* @default true
151+
*/
152+
extended: false
153+
}
154+
139155
export interface Exportable<T> {
140156
/**
141157
* A disambiguator to allow TypeScript to work out the type of the entry.
@@ -218,7 +234,7 @@ export interface Exportable<T> {
218234
* // `entries` contains the first 5 files/directories in the directory
219235
* ```
220236
*/
221-
content(options?: ExporterOptions): AsyncGenerator<T, void, unknown>
237+
content(options?: ExporterOptions | BasicExporterOptions): AsyncGenerator<T, void, unknown>
222238
}
223239

224240
/**
@@ -316,7 +332,39 @@ export interface Resolver { (cid: CID, name: string, path: string, toResolve: st
316332
export type UnixfsV1FileContent = AsyncIterable<Uint8Array> | Iterable<Uint8Array>
317333
export type UnixfsV1DirectoryContent = AsyncIterable<UnixFSEntry> | Iterable<UnixFSEntry>
318334
export type UnixfsV1Content = UnixfsV1FileContent | UnixfsV1DirectoryContent
319-
export interface UnixfsV1Resolver { (cid: CID, node: PBNode, unixfs: UnixFS, path: string, resolve: Resolve, depth: number, blockstore: ReadableStorage): (options: ExporterOptions) => UnixfsV1Content }
335+
336+
export interface UnixfsV1BasicContent {
337+
/**
338+
* The name of the entry
339+
*/
340+
name: string
341+
342+
/**
343+
* The path of the entry within the DAG in which it was encountered
344+
*/
345+
path: string
346+
347+
/**
348+
* The CID of the entry
349+
*/
350+
cid: CID
351+
352+
/**
353+
* Resolve the root node of the entry to parse the UnixFS metadata contained
354+
* there. The metadata will contain what kind of node it is (e.g. file,
355+
* directory, etc), the file size, and more.
356+
*/
357+
resolve(options?: AbortOptions): Promise<UnixFSEntry>
358+
}
359+
360+
export interface UnixFsV1ContentResolver {
361+
(options: ExporterOptions): UnixfsV1Content
362+
(options: BasicExporterOptions): UnixfsV1BasicContent
363+
}
364+
365+
export interface UnixfsV1Resolver {
366+
(cid: CID, node: PBNode, unixfs: UnixFS, path: string, resolve: Resolve, depth: number, blockstore: ReadableStorage): (options: ExporterOptions) => UnixfsV1Content
367+
}
320368

321369
export interface ShardTraversalContext {
322370
hamtDepth: number

packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/directory.ts

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,11 @@ import map from 'it-map'
33
import parallel from 'it-parallel'
44
import { pipe } from 'it-pipe'
55
import { CustomProgressEvent } from 'progress-events'
6-
import type { ExporterOptions, ExportWalk, UnixfsV1DirectoryContent, UnixfsV1Resolver } from '../../../index.js'
6+
import { isBasicExporterOptions } from '../../../utils/is-basic-exporter-options.ts'
7+
import type { BasicExporterOptions, ExporterOptions, ExportWalk, UnixFSEntry, UnixfsV1BasicContent, UnixfsV1Resolver } from '../../../index.js'
78

89
const directoryContent: UnixfsV1Resolver = (cid, node, unixfs, path, resolve, depth, blockstore) => {
9-
async function * yieldDirectoryContent (options: ExporterOptions = {}): UnixfsV1DirectoryContent {
10+
async function * yieldDirectoryContent (options: ExporterOptions | BasicExporterOptions = {}): any {
1011
const offset = options.offset ?? 0
1112
const length = options.length ?? node.Links.length
1213
const links = node.Links.slice(offset, length)
@@ -21,8 +22,24 @@ const directoryContent: UnixfsV1Resolver = (cid, node, unixfs, path, resolve, de
2122
return async () => {
2223
const linkName = link.Name ?? ''
2324
const linkPath = `${path}/${linkName}`
24-
const result = await resolve(link.Hash, linkName, linkPath, [], depth + 1, blockstore, options)
25-
return result.entry
25+
26+
const load = async (options = {}): Promise<UnixFSEntry> => {
27+
const result = await resolve(link.Hash, linkName, linkPath, [], depth + 1, blockstore, options)
28+
return result.entry
29+
}
30+
31+
if (isBasicExporterOptions(options)) {
32+
const basic: UnixfsV1BasicContent = {
33+
cid: link.Hash,
34+
name: linkName,
35+
path: linkPath,
36+
resolve: load
37+
}
38+
39+
return basic
40+
}
41+
42+
return load(options)
2643
}
2744
}),
2845
source => parallel(source, {

packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.ts

Lines changed: 28 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,12 @@ import parallel from 'it-parallel'
55
import { pipe } from 'it-pipe'
66
import { CustomProgressEvent } from 'progress-events'
77
import { NotUnixFSError } from '../../../errors.js'
8-
import type { ExporterOptions, Resolve, UnixfsV1DirectoryContent, UnixfsV1Resolver, ReadableStorage, ExportWalk } from '../../../index.js'
8+
import { isBasicExporterOptions } from '../../../utils/is-basic-exporter-options.ts'
9+
import type { ExporterOptions, Resolve, UnixfsV1DirectoryContent, UnixfsV1Resolver, ReadableStorage, ExportWalk, BasicExporterOptions, UnixFSEntry } from '../../../index.js'
910
import type { PBNode } from '@ipld/dag-pb'
1011

1112
const hamtShardedDirectoryContent: UnixfsV1Resolver = (cid, node, unixfs, path, resolve, depth, blockstore) => {
12-
function yieldHamtDirectoryContent (options: ExporterOptions = {}): UnixfsV1DirectoryContent {
13+
function yieldHamtDirectoryContent (options: ExporterOptions | BasicExporterOptions = {}): UnixfsV1DirectoryContent {
1314
options.onProgress?.(new CustomProgressEvent<ExportWalk>('unixfs:exporter:walk:hamt-sharded-directory', {
1415
cid
1516
}))
@@ -20,7 +21,7 @@ const hamtShardedDirectoryContent: UnixfsV1Resolver = (cid, node, unixfs, path,
2021
return yieldHamtDirectoryContent
2122
}
2223

23-
async function * listDirectory (node: PBNode, path: string, resolve: Resolve, depth: number, blockstore: ReadableStorage, options: ExporterOptions): UnixfsV1DirectoryContent {
24+
async function * listDirectory (node: PBNode, path: string, resolve: Resolve, depth: number, blockstore: ReadableStorage, options: ExporterOptions | BasicExporterOptions): any {
2425
const links = node.Links
2526

2627
if (node.Data == null) {
@@ -47,9 +48,28 @@ async function * listDirectory (node: PBNode, path: string, resolve: Resolve, de
4748
const name = link.Name != null ? link.Name.substring(padLength) : null
4849

4950
if (name != null && name !== '') {
50-
const result = await resolve(link.Hash, name, `${path}/${name}`, [], depth + 1, blockstore, options)
51+
const linkPath = `${path}/${name}`
52+
const load = async (options = {}): Promise<UnixFSEntry> => {
53+
const result = await resolve(link.Hash, name, linkPath, [], depth + 1, blockstore, options)
54+
return result.entry
55+
}
5156

52-
return { entries: result.entry == null ? [] : [result.entry] }
57+
if (isBasicExporterOptions(options)) {
58+
return {
59+
entries: [{
60+
cid: link.Hash,
61+
name,
62+
path: linkPath,
63+
resolve: load
64+
}]
65+
}
66+
}
67+
68+
return {
69+
entries: [
70+
await load()
71+
].filter(Boolean)
72+
}
5373
} else {
5474
// descend into subshard
5575
const block = await blockstore.get(link.Hash, options)
@@ -59,7 +79,9 @@ async function * listDirectory (node: PBNode, path: string, resolve: Resolve, de
5979
cid: link.Hash
6080
}))
6181

62-
return { entries: listDirectory(node, path, resolve, depth, blockstore, options) }
82+
return {
83+
entries: listDirectory(node, path, resolve, depth, blockstore, options)
84+
}
6385
}
6486
}
6587
}),
Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
import type { BasicExporterOptions } from '../index.js'
2+
3+
export function isBasicExporterOptions (obj?: any): obj is BasicExporterOptions {
4+
return obj?.extended === false
5+
}

packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.ts

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -363,4 +363,56 @@ describe('exporter sharded', function () {
363363
content: file?.node
364364
}]).to.deep.equal(files)
365365
})
366+
367+
it('exports basic sharded directory', async () => {
368+
const files: Record<string, { content: Uint8Array, cid?: CID }> = {}
369+
370+
// needs to result in a block that is larger than SHARD_SPLIT_THRESHOLD bytes
371+
for (let i = 0; i < 100; i++) {
372+
files[`file-${Math.random()}.txt`] = {
373+
content: uint8ArrayConcat(await all(randomBytes(100)))
374+
}
375+
}
376+
377+
const imported = await all(importer(Object.keys(files).map(path => ({
378+
path,
379+
content: asAsyncIterable(files[path].content)
380+
})), block, {
381+
wrapWithDirectory: true,
382+
shardSplitThresholdBytes: SHARD_SPLIT_THRESHOLD,
383+
rawLeaves: false
384+
}))
385+
386+
const dirCid = imported.pop()?.cid
387+
388+
if (dirCid == null) {
389+
throw new Error('No directory CID found')
390+
}
391+
392+
const exported = await exporter(dirCid, block)
393+
const dirFiles = await all(exported.content())
394+
395+
// delete shard contents
396+
for (const entry of dirFiles) {
397+
await block.delete(entry.cid)
398+
}
399+
400+
// list the contents again, this time just the basic version
401+
const basicDirFiles = await all(exported.content({
402+
extended: false
403+
}))
404+
expect(basicDirFiles.length).to.equal(dirFiles.length)
405+
406+
for (let i = 0; i < basicDirFiles.length; i++) {
407+
const dirFile = basicDirFiles[i]
408+
409+
expect(dirFile).to.have.property('name')
410+
expect(dirFile).to.have.property('path')
411+
expect(dirFile).to.have.property('cid')
412+
expect(dirFile).to.have.property('resolve')
413+
414+
// should fail because we have deleted this block
415+
await expect(dirFile.resolve()).to.eventually.be.rejected()
416+
}
417+
})
366418
})

packages/ipfs-unixfs-exporter/test/exporter.spec.ts

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1605,4 +1605,54 @@ describe('exporter', () => {
16051605

16061606
expect(actualInvocations).to.deep.equal(expectedInvocations)
16071607
})
1608+
1609+
it('exports basic directory', async () => {
1610+
const files: Record<string, { content: Uint8Array, cid?: CID }> = {}
1611+
1612+
for (let i = 0; i < 10; i++) {
1613+
files[`file-${Math.random()}.txt`] = {
1614+
content: uint8ArrayConcat(await all(randomBytes(100)))
1615+
}
1616+
}
1617+
1618+
const imported = await all(importer(Object.keys(files).map(path => ({
1619+
path,
1620+
content: asAsyncIterable(files[path].content)
1621+
})), block, {
1622+
wrapWithDirectory: true,
1623+
rawLeaves: false
1624+
}))
1625+
1626+
const dirCid = imported.pop()?.cid
1627+
1628+
if (dirCid == null) {
1629+
throw new Error('No directory CID found')
1630+
}
1631+
1632+
const exported = await exporter(dirCid, block)
1633+
const dirFiles = await all(exported.content())
1634+
1635+
// delete shard contents
1636+
for (const entry of dirFiles) {
1637+
await block.delete(entry.cid)
1638+
}
1639+
1640+
// list the contents again, this time just the basic version
1641+
const basicDirFiles = await all(exported.content({
1642+
extended: false
1643+
}))
1644+
expect(basicDirFiles.length).to.equal(dirFiles.length)
1645+
1646+
for (let i = 0; i < basicDirFiles.length; i++) {
1647+
const dirFile = basicDirFiles[i]
1648+
1649+
expect(dirFile).to.have.property('name')
1650+
expect(dirFile).to.have.property('path')
1651+
expect(dirFile).to.have.property('cid')
1652+
expect(dirFile).to.have.property('resolve')
1653+
1654+
// should fail because we have deleted this block
1655+
await expect(dirFile.resolve()).to.eventually.be.rejected()
1656+
}
1657+
})
16081658
})

0 commit comments

Comments
 (0)