fix: respect access mode and file open flags

sumimakito · sumimakito · commit fb54de0052e7 · 2025-02-22T20:17:37.000+08:00
diff --git a/packages/duckdb-wasm/src/bindings/bindings_base.ts b/packages/duckdb-wasm/src/bindings/bindings_base.ts
@@ -1,5 +1,5 @@
 import { DuckDBModule, PThread } from './duckdb_module';
-import { DuckDBConfig } from './config';
+import { DuckDBAccessMode, DuckDBConfig } from './config';
 import { Logger } from '../log';
 import { InstantiationProgress } from './progress';
 import { DuckDBBindings } from './bindings_interface';
@@ -469,9 +469,9 @@ export abstract class DuckDBBindingsBase implements DuckDBBindings {
         }
         dropResponseBuffers(this.mod);
     }
-    public async prepareFileHandle(fileName: string, protocol: DuckDBDataProtocol): Promise<void> {
+    public async prepareFileHandle(fileName: string, protocol: DuckDBDataProtocol, accessMode?: DuckDBAccessMode): Promise<void> {
         if (protocol === DuckDBDataProtocol.BROWSER_FSACCESS && this._runtime.prepareFileHandles) {
-            const list = await this._runtime.prepareFileHandles([fileName], DuckDBDataProtocol.BROWSER_FSACCESS);
+            const list = await this._runtime.prepareFileHandles([fileName], DuckDBDataProtocol.BROWSER_FSACCESS, accessMode);
             for (const item of list) {
                 const { handle, path: filePath, fromCached } = item;
                 if (!fromCached && handle.getSize()) {
@@ -483,9 +483,9 @@ export abstract class DuckDBBindingsBase implements DuckDBBindings {
         throw new Error(`prepareFileHandle: unsupported protocol ${protocol}`);
     }
     /** Prepare a file handle that could only be acquired aschronously */
-    public async prepareDBFileHandle(path: string, protocol: DuckDBDataProtocol): Promise<void> {
+    public async prepareDBFileHandle(path: string, protocol: DuckDBDataProtocol, accessMode?: DuckDBAccessMode): Promise<void> {
         if (protocol === DuckDBDataProtocol.BROWSER_FSACCESS && this._runtime.prepareDBFileHandle) {
-            const list = await this._runtime.prepareDBFileHandle(path, DuckDBDataProtocol.BROWSER_FSACCESS);
+            const list = await this._runtime.prepareDBFileHandle(path, DuckDBDataProtocol.BROWSER_FSACCESS, accessMode);
             for (const item of list) {
                 const { handle, path: filePath, fromCached } = item;
                 if (!fromCached && handle.getSize()) {
diff --git a/packages/duckdb-wasm/src/bindings/bindings_interface.ts b/packages/duckdb-wasm/src/bindings/bindings_interface.ts
@@ -1,4 +1,4 @@
-import { DuckDBConfig, DuckDBConnection, DuckDBDataProtocol, FileStatistics, InstantiationProgress } from '.';
+import { DuckDBAccessMode, DuckDBConfig, DuckDBConnection, DuckDBDataProtocol, FileStatistics, InstantiationProgress } from '.';
 import { CSVInsertOptions, JSONInsertOptions, ArrowInsertOptions } from './insert_options';
 import { ScriptTokens } from './tokens';
 import { WebFile } from './web_file';
@@ -54,8 +54,8 @@ export interface DuckDBBindings {
         protocol: DuckDBDataProtocol,
         directIO: boolean,
     ): Promise<HandleType>;
-    prepareFileHandle(path: string, protocol: DuckDBDataProtocol): Promise<void>;
-    prepareDBFileHandle(path: string, protocol: DuckDBDataProtocol): Promise<void>;
+    prepareFileHandle(path: string, protocol: DuckDBDataProtocol, accessMode?: DuckDBAccessMode): Promise<void>;
+    prepareDBFileHandle(path: string, protocol: DuckDBDataProtocol, accessMode?: DuckDBAccessMode): Promise<void>;
     globFiles(path: string): WebFile[];
     dropFile(name: string): void;
     dropFiles(): void;
diff --git a/packages/duckdb-wasm/src/bindings/runtime.ts b/packages/duckdb-wasm/src/bindings/runtime.ts
@@ -1,3 +1,4 @@
+import { DuckDBAccessMode } from './config';
 import { DuckDBModule } from './duckdb_module';
 import { UDFFunction } from './udf_function';
 import * as udf_rt from './udf_runtime';
@@ -58,6 +59,16 @@ export enum FileFlags {
     FILE_FLAGS_FILE_CREATE_NEW = 1 << 4,
     //! Open file in append mode
     FILE_FLAGS_APPEND = 1 << 5,
+    //! Open file with restrictive permissions (600 on linux/mac) can only be used when creating, throws if file exists
+    FILE_FLAGS_PRIVATE = 1 << 6,
+    //! Return NULL if the file does not exist instead of throwing an error
+    FILE_FLAGS_NULL_IF_NOT_EXISTS = 1 << 7,
+    //! Multiple threads may perform reads and writes in parallel
+    FILE_FLAGS_PARALLEL_ACCESS = 1 << 8,
+    //! Ensure that this call creates the file, throw is file exists
+    FILE_FLAGS_EXCLUSIVE_CREATE = 1 << 9,
+    //!  Return NULL if the file exist instead of throwing an error
+    FILE_FLAGS_NULL_IF_EXISTS = 1 << 10,
 }
 
 /** Configuration for the AWS S3 Filesystem */
@@ -158,8 +169,8 @@ export interface DuckDBRuntime {
 
     // Prepare a file handle that could only be acquired aschronously
     prepareFileHandle?: (path: string, protocol: DuckDBDataProtocol) => Promise<PreparedDBFileHandle[]>;
-    prepareFileHandles?: (path: string[], protocol: DuckDBDataProtocol) => Promise<PreparedDBFileHandle[]>;
-    prepareDBFileHandle?: (path: string, protocol: DuckDBDataProtocol) => Promise<PreparedDBFileHandle[]>;
+    prepareFileHandles?: (path: string[], protocol: DuckDBDataProtocol, accessMode?: DuckDBAccessMode) => Promise<PreparedDBFileHandle[]>;
+    prepareDBFileHandle?: (path: string, protocol: DuckDBDataProtocol, accessMode?: DuckDBAccessMode) => Promise<PreparedDBFileHandle[]>;
 
     // Call a scalar UDF function
     callScalarUDF(
diff --git a/packages/duckdb-wasm/src/bindings/runtime_browser.ts b/packages/duckdb-wasm/src/bindings/runtime_browser.ts
@@ -15,6 +15,7 @@ import {
 } from './runtime';
 import { DuckDBModule } from './duckdb_module';
 import * as udf from './udf_runtime';
+import { DuckDBAccessMode } from './config';
 
 const OPFS_PREFIX_LEN = 'opfs://'.length;
 const PATH_SEP_REGEX = /\/|\\/;
@@ -110,8 +111,11 @@ export const BROWSER_RUNTIME: DuckDBRuntime & {
             BROWSER_RUNTIME._opfsRoot = await navigator.storage.getDirectory();
 	}
     },
-    /** Prepare a file handle that could only be acquired aschronously */
-    async prepareFileHandles(filePaths: string[], protocol: DuckDBDataProtocol): Promise<PreparedDBFileHandle[]> {
+    /** Prepare a file handle that could only be acquired asynchronously */
+    async prepareFileHandles(filePaths: string[], protocol: DuckDBDataProtocol, accessMode?: DuckDBAccessMode): Promise<PreparedDBFileHandle[]> {
+        // DuckDBAccessMode.UNDEFINED will be treated as READ_WRITE
+        // See: https://github.com/duckdb/duckdb/blob/5f5512b827df6397afd31daedb4bbdee76520019/src/main/database.cpp#L442-L444
+        const isReadWrite = !accessMode || accessMode === DuckDBAccessMode.READ_WRITE;
         if (protocol === DuckDBDataProtocol.BROWSER_FSACCESS) {
             await BROWSER_RUNTIME.assignOPFSRoot();
             const prepare = async (path: string): Promise<PreparedDBFileHandle> => {
@@ -135,13 +139,16 @@ export const BROWSER_RUNTIME: DuckDBRuntime & {
                     }
                     // mkdir -p
                     for (const folder of folders) {
-                        dirHandle = await dirHandle.getDirectoryHandle(folder, { create: true });
+                        dirHandle = await dirHandle.getDirectoryHandle(folder, { create: isReadWrite });
                     }
                 }
                 const fileHandle = await dirHandle.getFileHandle(fileName, { create: false }).catch(e => {
                     if (e?.name === 'NotFoundError') {
-                        console.debug(`File ${path} does not exists yet, creating...`);
-                        return dirHandle.getFileHandle(fileName, { create: true });
+                        if (isReadWrite) {
+                            console.debug(`File ${path} does not exists yet, creating...`);
+                            return dirHandle.getFileHandle(fileName, { create: true });
+                        }
+                        console.debug(`File ${path} does not exists, aborting as we are in read-only mode`);
                     }
                     throw e;
                 });
@@ -166,11 +173,11 @@ export const BROWSER_RUNTIME: DuckDBRuntime & {
         }
         throw new Error(`Unsupported protocol ${protocol} for paths ${filePaths} with protocol ${protocol}`);
     },
-    /** Prepare a file handle that could only be acquired aschronously */
-    async prepareDBFileHandle(dbPath: string, protocol: DuckDBDataProtocol): Promise<PreparedDBFileHandle[]> {
+    /** Prepare a file handle that could only be acquired asynchronously */
+    async prepareDBFileHandle(dbPath: string, protocol: DuckDBDataProtocol, accessMode?: DuckDBAccessMode): Promise<PreparedDBFileHandle[]> {
         if (protocol === DuckDBDataProtocol.BROWSER_FSACCESS && this.prepareFileHandles) {
             const filePaths = [dbPath, `${dbPath}.wal`];
-            return this.prepareFileHandles(filePaths, protocol);
+            return this.prepareFileHandles(filePaths, protocol, accessMode);
         }
         throw new Error(`Unsupported protocol ${protocol} for path ${dbPath} with protocol ${protocol}`);
     },
diff --git a/packages/duckdb-wasm/src/bindings/runtime_node.ts b/packages/duckdb-wasm/src/bindings/runtime_node.ts
@@ -74,16 +74,32 @@ export const NODE_RUNTIME: DuckDBRuntime & {
             switch (file?.dataProtocol) {
                 // Native file
                 case DuckDBDataProtocol.NODE_FS: {
+                    let openFlags = fs.constants.O_RDONLY;
+                    if (flags & FileFlags.FILE_FLAGS_WRITE) {
+                        openFlags = fs.constants.O_RDWR;
+                    }
+                    if (flags & FileFlags.FILE_FLAGS_FILE_CREATE) {
+                        openFlags |= fs.constants.O_CREAT;
+                    } else if (flags & FileFlags.FILE_FLAGS_FILE_CREATE_NEW) {
+                        openFlags |= fs.constants.O_TRUNC;
+                    }
                     let fd = NODE_RUNTIME._files?.get(file.dataUrl!);
-                    if (fd === null || fd === undefined) {
-                        fd = fs.openSync(
-                            file.dataUrl!,
-                            fs.constants.O_CREAT | fs.constants.O_RDWR,
-                            fs.constants.S_IRUSR | fs.constants.S_IWUSR,
-                        );
-                        NODE_RUNTIME._filesById?.set(file.fileId!, fd);
+                    let fileSize = 0;
+                    try {
+                        if (fd === null || fd === undefined) {
+                            fd = fs.openSync(file.dataUrl!, openFlags, fs.constants.S_IRUSR | fs.constants.S_IWUSR);
+                            NODE_RUNTIME._filesById?.set(file.fileId!, fd);
+                        }
+                        fileSize = fs.fstatSync(fd).size;
+                    }
+                    catch (e: any) {
+                        if (e.code === 'ENOENT' && (flags & FileFlags.FILE_FLAGS_NULL_IF_NOT_EXISTS)) {
+                            // No-op because we intend to ignore ENOENT while the file does not exist
+                            return 0; // nullptr
+                        } else {
+                            throw e;
+                        }
                     }
-                    const fileSize = fs.fstatSync(fd).size;
                     const result = mod._malloc(2 * 8);
                     mod.HEAPF64[(result >> 3) + 0] = +fileSize;
                     mod.HEAPF64[(result >> 3) + 1] = 0;
diff --git a/packages/duckdb-wasm/src/parallel/worker_dispatcher.ts b/packages/duckdb-wasm/src/parallel/worker_dispatcher.ts
@@ -136,8 +136,9 @@ export abstract class AsyncDuckDBDispatcher implements Logger {
 
                 case WorkerRequestType.OPEN: {
                     const path = request.data.path;
+                    const accessMode = request.data.accessMode;
                     if (path?.startsWith('opfs://')) {
-                        await this._bindings.prepareDBFileHandle(path, DuckDBDataProtocol.BROWSER_FSACCESS);
+                        await this._bindings.prepareDBFileHandle(path, DuckDBDataProtocol.BROWSER_FSACCESS, accessMode);
                         request.data.useDirectIO = true;
                     }
                     this._bindings.open(request.data);
diff --git a/packages/duckdb-wasm/test/index_node.ts b/packages/duckdb-wasm/test/index_node.ts
@@ -69,6 +69,7 @@ import { testAllTypes, testAllTypesAsync } from './all_types.test';
 import { testBindings, testAsyncBindings } from './bindings.test';
 import { testBatchStream } from './batch_stream.test';
 import { testFilesystem } from './filesystem.test';
+import { testNodeFS } from './nodefs.test';
 import { testAsyncBatchStream } from './batch_stream_async.test';
 import { testArrowInsert, testArrowInsertAsync } from './insert_arrow.test';
 import { testJSONInsert, testJSONInsertAsync } from './insert_json.test';
@@ -92,6 +93,7 @@ testAsyncBindings(() => adb!, dataDir, duckdb.DuckDBDataProtocol.NODE_FS);
 testBatchStream(() => db!);
 testAsyncBatchStream(() => adb!);
 testFilesystem(() => adb!, resolveData, dataDir, duckdb.DuckDBDataProtocol.NODE_FS);
+testNodeFS(() => adb!);
 testArrowInsert(() => db!);
 testArrowInsertAsync(() => adb!);
 testJSONInsert(() => db!);
diff --git a/packages/duckdb-wasm/test/nodefs.test.ts b/packages/duckdb-wasm/test/nodefs.test.ts
@@ -0,0 +1,71 @@
+import * as duckdb from '../src/';
+import { tmpdir } from 'os';
+import { randomUUID } from 'crypto';
+import path from 'path';
+import { unlink } from 'fs/promises';
+
+export function testNodeFS(db: () => duckdb.AsyncDuckDB): void {
+    const files: string[] = [];
+
+    afterAll(async () => {
+        await Promise.all(files.map(file => unlink(file).catch(() => {})));
+        await db().flushFiles();
+        await db().dropFiles();
+    });
+
+    describe('Node FS', () => {
+        it('Should not create an empty DB file in read-only mode for non-existent path', async () => {
+            const tmp = tmpdir();
+            const filename = `duckdb_test_${randomUUID().replace(/-/g, '')}`;
+            files.push(path.join(tmp, filename));
+
+            await expectAsync(
+                db().open({
+                    path: path.join(tmp, filename),
+                    accessMode: duckdb.DuckDBAccessMode.READ_ONLY,
+                }),
+            ).toBeRejectedWithError(/database does not exist/);
+        });
+
+        it('Should create DB file in read-write mode for non-existent path', async () => {
+            const tmp = tmpdir();
+            const filename = `duckdb_test_${randomUUID().replace(/-/g, '')}`;
+            files.push(path.join(tmp, filename));
+
+            await expectAsync(
+                db().open({
+                    path: path.join(tmp, filename),
+                    accessMode: duckdb.DuckDBAccessMode.READ_WRITE,
+                }),
+            ).toBeResolved();
+        });
+
+        it('Should create an empty DB file in read-only mode for non-existent path with direct I/O', async () => {
+            const tmp = tmpdir();
+            const filename = `duckdb_test_${randomUUID().replace(/-/g, '')}`;
+            files.push(path.join(tmp, filename));
+
+            await expectAsync(
+                db().open({
+                    path: path.join(tmp, filename),
+                    accessMode: duckdb.DuckDBAccessMode.READ_ONLY,
+                    useDirectIO: true,
+                }),
+            ).toBeRejectedWithError(/database does not exist/);
+        });
+
+        it('Should create DB file in read-write mode for non-existent path with direct I/O', async () => {
+            const tmp = tmpdir();
+            const filename = `duckdb_test_${randomUUID().replace(/-/g, '')}`;
+            files.push(path.join(tmp, filename));
+
+            await expectAsync(
+                db().open({
+                    path: path.join(tmp, filename),
+                    accessMode: duckdb.DuckDBAccessMode.READ_WRITE,
+                    useDirectIO: true,
+                }),
+            ).toBeResolved();
+        });
+    });
+}
diff --git a/packages/duckdb-wasm/test/opfs.test.ts b/packages/duckdb-wasm/test/opfs.test.ts
diff --git a/patches/duckdb/fix_load_database.patch b/patches/duckdb/fix_load_database.patch
