From 9f28519e329938a244304e0a6b56f386ba340a49 Mon Sep 17 00:00:00 2001 From: Votre Nom Date: Thu, 19 Jun 2025 03:35:57 +0200 Subject: [PATCH 1/4] Add async iterator --- api/src/DuckDBResult.ts | 31 +++++++++ api/test/async-iterator.test.ts | 117 ++++++++++++++++++++++++++++++++ 2 files changed, 148 insertions(+) create mode 100644 api/test/async-iterator.test.ts diff --git a/api/src/DuckDBResult.ts b/api/src/DuckDBResult.ts index 2f749fb..2eea62d 100644 --- a/api/src/DuckDBResult.ts +++ b/api/src/DuckDBResult.ts @@ -239,4 +239,35 @@ export class DuckDBResult { public async getRowObjectsJson(): Promise[]> { return this.convertRowObjects(JsonDuckDBValueConverter); } + + /** + * Async iterator that yields rows one by one as objects + * Usage: for await (const row of result) { ... } + */ + public async *[Symbol.asyncIterator](): AsyncIterableIterator> { + const columnNames = this.deduplicatedColumnNames(); + while (true) { + const chunk = await this.fetchChunk(); + if (!chunk || chunk.rowCount === 0) { + break; + } + // Pre-fetch all column vectors for this chunk for efficiency + const columnVectors = []; + for (let colIndex = 0; colIndex < this.columnCount; colIndex++) { + columnVectors.push(chunk.getColumnVector(colIndex)); + } + // Yield each row in the chunk + for (let rowIndex = 0; rowIndex < chunk.rowCount; rowIndex++) { + const row: Record = {}; + + for (let colIndex = 0; colIndex < this.columnCount; colIndex++) { + const columnName = columnNames[colIndex]; + const value = columnVectors[colIndex].getItem(rowIndex); + row[columnName] = JSDuckDBValueConverter(value, columnVectors[colIndex].type, JSDuckDBValueConverter); + } + + yield row; + } + } + } } diff --git a/api/test/async-iterator.test.ts b/api/test/async-iterator.test.ts new file mode 100644 index 0000000..95e106f --- /dev/null +++ b/api/test/async-iterator.test.ts @@ -0,0 +1,117 @@ +import { describe, test, assert } from 'vitest'; +import { withConnection } from './api.test'; + +describe('DuckDBResult async iterator', () => { + + test('should iterate over basic query results', async () => { + await withConnection(async (connection) => { + await connection.run('CREATE TABLE test_table (id INTEGER, name VARCHAR)'); + await connection.run("INSERT INTO test_table VALUES (1, 'Alice'), (2, 'Bob')"); + + const result = await connection.run('SELECT * FROM test_table ORDER BY id'); + + const rows: any[] = []; + for await (const row of result) { + rows.push(row); + } + + assert.equal(rows.length, 2); + assert.deepEqual(rows[0], { id: 1, name: 'Alice' }); + assert.deepEqual(rows[1], { id: 2, name: 'Bob' }); + }); + }); + + test('should handle empty result sets', async () => { + await withConnection(async (connection) => { + await connection.run('CREATE TABLE empty_table (id INTEGER)'); + + const result = await connection.run('SELECT * FROM empty_table'); + + const rows: any[] = []; + for await (const row of result) { + rows.push(row); + } + + assert.equal(rows.length, 0); + }); + }); + + test('should allow early termination with break', async () => { + await withConnection(async (connection) => { + await connection.run('CREATE TABLE numbers (value INTEGER)'); + await connection.run('INSERT INTO numbers SELECT range FROM range(10)'); + + const result = await connection.run('SELECT * FROM numbers ORDER BY value'); + + const rows: any[] = []; + for await (const row of result) { + rows.push(row); + if (rows.length >= 3) { + break; + } + } + + assert.equal(rows.length, 3); + assert.equal(rows[0].value, 0); + assert.equal(rows[2].value, 2); + }); + }); + + test('should handle null values', async () => { + await withConnection(async (connection) => { + await connection.run('CREATE TABLE nullable_table (id INTEGER, value VARCHAR)'); + await connection.run("INSERT INTO nullable_table VALUES (1, 'test'), (2, null)"); + + const result = await connection.run('SELECT * FROM nullable_table ORDER BY id'); + + const rows: any[] = []; + for await (const row of result) { + rows.push(row); + } + + assert.equal(rows.length, 2); + assert.equal(rows[0].value, 'test'); + assert.isNull(rows[1].value); + }); + }); + + test('should fetch chunks progressively', async () => { + await withConnection(async (connection) => { + const result = await connection.run('SELECT range as value FROM range(10000)'); + + let count = 0; + let chunkCount = 0; + + // Track chunks + const originalFetchChunk = result.fetchChunk.bind(result); + result.fetchChunk = async function () { + const chunk = await originalFetchChunk(); + if (chunk && chunk.rowCount > 0) { + chunkCount++; + } + return chunk; + }; + + assert.equal(chunkCount, 0); + + // Use the SAME iterator instance + const iterator = result[Symbol.asyncIterator](); + + // Get first row + const firstResult = await iterator.next(); + assert.equal(chunkCount, 1); + assert.isFalse(firstResult.done); + count++; + + // Continue with the same iterator + let nextResult = await iterator.next(); + while (!nextResult.done) { + count++; + nextResult = await iterator.next(); + } + + assert.equal(count, 10000); + assert.isAbove(chunkCount, 1); + }); + }); +}); \ No newline at end of file From fc25c2c985e5c6d2ad409e3cd6f1b5bee3a1a937 Mon Sep 17 00:00:00 2001 From: Votre Nom Date: Thu, 19 Jun 2025 03:36:31 +0200 Subject: [PATCH 2/4] Fix async iterator test imports --- api/test/async-iterator.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/api/test/async-iterator.test.ts b/api/test/async-iterator.test.ts index 95e106f..38b027a 100644 --- a/api/test/async-iterator.test.ts +++ b/api/test/async-iterator.test.ts @@ -1,5 +1,6 @@ import { describe, test, assert } from 'vitest'; import { withConnection } from './api.test'; +import assert from 'assert'; describe('DuckDBResult async iterator', () => { From c82f3929aae4d25b83ac40fb6c80121d1a0f3073 Mon Sep 17 00:00:00 2001 From: Votre Nom Date: Thu, 19 Jun 2025 03:51:44 +0200 Subject: [PATCH 3/4] fixes --- api/test/api.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/test/api.test.ts b/api/test/api.test.ts index 1b35eff..a540484 100644 --- a/api/test/api.test.ts +++ b/api/test/api.test.ts @@ -140,7 +140,7 @@ async function sleep(ms: number): Promise { }); } -async function withConnection( +export async function withConnection( fn: (connection: DuckDBConnection) => Promise ) { const instance = await DuckDBInstance.create(); From 6d148ef0f7908d9c2d7e37c9dd68f2d2b658c0be Mon Sep 17 00:00:00 2001 From: Votre Nom Date: Thu, 19 Jun 2025 16:25:18 +0200 Subject: [PATCH 4/4] remove double assert import --- api/test/async-iterator.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/api/test/async-iterator.test.ts b/api/test/async-iterator.test.ts index 38b027a..95e106f 100644 --- a/api/test/async-iterator.test.ts +++ b/api/test/async-iterator.test.ts @@ -1,6 +1,5 @@ import { describe, test, assert } from 'vitest'; import { withConnection } from './api.test'; -import assert from 'assert'; describe('DuckDBResult async iterator', () => {