feat: enable custom file/directory builders (#413)

To allow adding building custom file/directory IPLD objects, two additional fields to `DagBuilderOptions` & `ImporterOptions` are added that would override the `dirBuilder` & `fileBuilder` with other implementations.

---------

Co-authored-by: achingbrain <[email protected]>

Author: clostao

packages/ipfs-unixfs-importer/src/dag-builder/dir.ts

@@ -9,7 +9,11 @@ export interface DirBuilderOptions {
   signal?: AbortSignal
 }
 
-export const dirBuilder = async (dir: Directory, blockstore: WritableStorage, options: DirBuilderOptions): Promise<InProgressImportResult> => {
+export interface DirBuilder {
+  (dir: Directory, blockstore: WritableStorage, options: DirBuilderOptions): Promise<InProgressImportResult>
+}
+
+export const defaultDirBuilder: DirBuilder = async (dir: Directory, blockstore: WritableStorage, options: DirBuilderOptions): Promise<InProgressImportResult> => {
   const unixfs = new UnixFS({
     type: 'directory',
     mtime: dir.mtime,

packages/ipfs-unixfs-importer/src/dag-builder/file.ts

@@ -30,7 +30,7 @@ async function * buildFileBatch (file: File, blockstore: WritableStorage, option
       }
 
       continue
-    } else if (count === 1 && (previous != null)) {
+    } else if (count === 1 && previous != null) {
       // we have the second block of a multiple block import so yield the first
       yield {
         ...previous,
@@ -131,7 +131,7 @@ const reduce = (file: File, blockstore: WritableStorage, options: ReduceOptions)
           return true
         }
 
-        if ((leaf.unixfs != null) && (leaf.unixfs.data == null) && leaf.unixfs.fileSize() > 0n) {
+        if (leaf.unixfs != null && leaf.unixfs.data == null && leaf.unixfs.fileSize() > 0n) {
           return true
         }
 
@@ -189,10 +189,17 @@ const reduce = (file: File, blockstore: WritableStorage, options: ReduceOptions)
   return reducer
 }
 
+export interface FileBuilder {
+  (file: File, blockstore: WritableStorage, options: FileBuilderOptions): Promise<InProgressImportResult>
+}
+
 export interface FileBuilderOptions extends BuildFileBatchOptions, ReduceOptions {
   layout: FileLayout
 }
 
-export const fileBuilder = async (file: File, block: WritableStorage, options: FileBuilderOptions): Promise<InProgressImportResult> => {
-  return options.layout(buildFileBatch(file, block, options), reduce(file, block, options))
+export const defaultFileBuilder: FileBuilder = async (file: File, block: WritableStorage, options: FileBuilderOptions): Promise<InProgressImportResult> => {
+  return options.layout(
+    buildFileBatch(file, block, options),
+    reduce(file, block, options)
+  )
 }

packages/ipfs-unixfs-importer/src/dag-builder/index.ts

@@ -1,9 +1,9 @@
 import { CustomProgressEvent } from 'progress-events'
 import { InvalidContentError } from '../errors.js'
-import { dirBuilder } from './dir.js'
-import { fileBuilder } from './file.js'
-import type { DirBuilderOptions } from './dir.js'
-import type { FileBuilderOptions } from './file.js'
+import { defaultDirBuilder } from './dir.js'
+import { defaultFileBuilder } from './file.js'
+import type { DirBuilder, DirBuilderOptions } from './dir.js'
+import type { FileBuilder, FileBuilderOptions } from './file.js'
 import type { ChunkValidator } from './validate-chunks.js'
 import type { Chunker } from '../chunker/index.js'
 import type { Directory, File, FileCandidate, ImportCandidate, ImporterProgressEvents, InProgressImportResult, WritableStorage } from '../index.js'
@@ -45,11 +45,11 @@ function contentAsAsyncIterable (content: Uint8Array | AsyncIterable<Uint8Array>
     if (content instanceof Uint8Array) {
       return (async function * () {
         yield content
-      }())
+      })()
     } else if (isIterable(content)) {
       return (async function * () {
         yield * content
-      }())
+      })()
     } else if (isAsyncIterable(content)) {
       return content
     }
@@ -64,6 +64,8 @@ export interface DagBuilderOptions extends FileBuilderOptions, DirBuilderOptions
   chunker: Chunker
   chunkValidator: ChunkValidator
   wrapWithDirectory: boolean
+  dirBuilder?: DirBuilder
+  fileBuilder?: FileBuilder
 }
 
 export type ImporterSourceStream = AsyncIterable<ImportCandidate> | Iterable<ImportCandidate>
@@ -109,6 +111,8 @@ export function defaultDagBuilder (options: DagBuilderOptions): DAGBuilder {
           originalPath
         }
 
+        const fileBuilder = options.fileBuilder ?? defaultFileBuilder
+
         yield async () => fileBuilder(file, blockstore, options)
       } else if (entry.path != null) {
         const dir: Directory = {
@@ -118,6 +122,8 @@ export function defaultDagBuilder (options: DagBuilderOptions): DAGBuilder {
           originalPath
         }
 
+        const dirBuilder = options.dirBuilder ?? defaultDirBuilder
+
         yield async () => dirBuilder(dir, blockstore, options)
       } else {
         throw new Error('Import candidate must have content or path or both')

packages/ipfs-unixfs-importer/src/index.ts

@@ -73,7 +73,8 @@ import { balanced } from './layout/index.js'
 import { defaultTreeBuilder } from './tree-builder.js'
 import type { Chunker } from './chunker/index.js'
 import type { BufferImportProgressEvents } from './dag-builder/buffer-importer.js'
-import type { ReducerProgressEvents } from './dag-builder/file.js'
+import type { DirBuilder } from './dag-builder/dir.js'
+import type { FileBuilder, ReducerProgressEvents } from './dag-builder/file.js'
 import type { DAGBuilder, DagBuilderProgressEvents } from './dag-builder/index.js'
 import type { ChunkValidator } from './dag-builder/validate-chunks.js'
 import type { FileLayout } from './layout/index.js'
@@ -276,6 +277,20 @@ export interface ImporterOptions extends ProgressOptions<ImporterProgressEvents>
    * `Error`
    */
   chunkValidator?: ChunkValidator
+
+  /**
+   * This option can be used to override how a directory IPLD node is built.
+   *
+   * This function takes a `Directory` object and returns a `Promise` that resolves to an `InProgressImportResult`.
+   */
+  dirBuilder?: DirBuilder
+
+  /**
+   * This option can be used to override how a file IPLD node is built.
+   *
+   * This function takes a `File` object and returns a `Promise` that resolves to an `InProgressImportResult`.
+   */
+  fileBuilder?: FileBuilder
 }
 
 export type ImportCandidateStream = AsyncIterable<FileCandidate | DirectoryCandidate> | Iterable<FileCandidate | DirectoryCandidate>
@@ -342,7 +357,9 @@ export async function * importer (source: ImportCandidateStream, blockstore: Wri
     blockWriteConcurrency,
     reduceSingleLeafToSelf,
     cidVersion,
-    onProgress: options.onProgress
+    onProgress: options.onProgress,
+    dirBuilder: options.dirBuilder,
+    fileBuilder: options.fileBuilder
   })
   const buildTree: TreeBuilder = options.treeBuilder ?? defaultTreeBuilder({
     wrapWithDirectory,

packages/ipfs-unixfs-importer/test/custom-dag-builder-params.spec.ts

@@ -0,0 +1,40 @@
+import { expect } from 'aegir/chai'
+import { MemoryBlockstore } from 'blockstore-core'
+import { defaultDirBuilder } from '../src/dag-builder/dir.js'
+import { defaultFileBuilder } from '../src/dag-builder/file.js'
+import { importer } from '../src/index.js'
+import type { DirBuilder } from '../src/dag-builder/dir.js'
+import type { FileBuilder } from '../src/dag-builder/file.js'
+
+describe('CustomParamsDagBuilder', () => {
+  it('should build a dag with custom dir builder', async () => {
+    const counter = { dirCounter: 0, fileCounter: 0 }
+    const customDirBuilder: DirBuilder = async (...args) => {
+      counter.dirCounter++
+      return defaultDirBuilder(...args)
+    }
+
+    const customFileBuilder: FileBuilder = async (...args) => {
+      counter.fileCounter++
+      return defaultFileBuilder(...args)
+    }
+
+    const blockstore = new MemoryBlockstore()
+    const files = []
+    for await (const file of importer([{
+      path: './src/file.txt',
+      content: new Uint8Array(
+        'hello world'.split('').map((char) => char.charCodeAt(0))
+      )
+    }, {
+      path: './src'
+    }], blockstore, {
+      dirBuilder: customDirBuilder,
+      fileBuilder: customFileBuilder
+    })) {
+      files.push(file)
+    }
+
+    expect(counter.dirCounter).to.equal(1)
+  })
+})