chore: refactor tablewisefiles to table (#152)

vpbs2 · web-flow · commit 640af75bc3ad · 2025-07-04T10:42:53.000+05:30
* refactor prequery

* fix: test

* update: test
diff --git a/benchmarking/public/runner/memory-runner.html b/benchmarking/public/runner/memory-runner.html
diff --git a/benchmarking/src/app/dbm-context/parallel-indexed-dbm-context.tsx b/benchmarking/src/app/dbm-context/parallel-indexed-dbm-context.tsx
@@ -2,9 +2,9 @@ import {
   DBMParallel,
   IFrameRunnerManager,
   ParallelIndexedDBFileManager,
+  Table,
 } from '@devrev/meerkat-dbm';
 import log from 'loglevel';
-import { TableWiseFiles } from 'meerkat-dbm/src/types/common-types';
 import { useRef, useState } from 'react';
 import { DBMContext } from '../hooks/dbm-context';
 import { useClassicEffect } from '../hooks/use-classic-effect';
@@ -45,14 +45,14 @@ export const ParallelIndexedDBMProvider = ({
       fetchTableFileBuffers: async (table) => {
         return [];
       },
-      fetchPreQuery: (runnerId: string, tableWiseFiles: TableWiseFiles[]) => {
+      fetchPreQuery: (runnerId: string, tables: Table[]) => {
         const preQueries: string[] = [];
 
-        for (const tableWiseFile of tableWiseFiles) {
+        for (const tableData of tables) {
           preQueries.push(
             generateViewQuery(
-              tableWiseFile.tableName,
-              tableWiseFile.files.map((file) => file.fileName)
+              tableData.tableName,
+              tableData.files.map((file) => file.fileName)
             )
           );
         }
diff --git a/benchmarking/src/app/dbm-context/parallel-memory-dbm-context.tsx b/benchmarking/src/app/dbm-context/parallel-memory-dbm-context.tsx
@@ -2,9 +2,9 @@ import {
   DBMParallel,
   IFrameRunnerManager,
   ParallelMemoryFileManager,
+  Table,
 } from '@devrev/meerkat-dbm';
 import log from 'loglevel';
-import { TableWiseFiles } from 'meerkat-dbm/src/types/common-types';
 import { useRef, useState } from 'react';
 import { DBMContext } from '../hooks/dbm-context';
 import { useClassicEffect } from '../hooks/use-classic-effect';
@@ -45,14 +45,14 @@ export const ParallelMemoryDBMProvider = ({
       fetchTableFileBuffers: async (table) => {
         return fileManagerRef.current.getTableBufferData(table);
       },
-      fetchPreQuery: (runnerId: string, tableWiseFiles: TableWiseFiles[]) => {
+      fetchPreQuery: (runnerId: string, tables: Table[]) => {
         const preQueries: string[] = [];
 
-        for (const tableWiseFile of tableWiseFiles) {
+        for (const tableData of tables) {
           preQueries.push(
             generateViewQuery(
-              tableWiseFile.tableName,
-              tableWiseFile.files.map((file) => file.fileName)
+              tableData.tableName,
+              tableData.files.map((file) => file.fileName)
             )
           );
         }
diff --git a/benchmarking/src/app/query-benchmarking/query-benchmarking.tsx b/benchmarking/src/app/query-benchmarking/query-benchmarking.tsx
@@ -1,4 +1,4 @@
-import { TableWiseFiles } from 'meerkat-dbm/src/types';
+import { Table } from '@devrev/meerkat-dbm';
 import { useMemo, useState } from 'react';
 import { TEST_QUERIES } from '../constants';
 import { useDBM } from '../hooks/dbm-context';
@@ -17,8 +17,8 @@ export const QueryBenchmarking = () => {
 
   const preQuery = useMemo(
     () =>
-      async (tablesFileData: TableWiseFiles[]): Promise<void> => {
-        for (const table of tablesFileData) {
+      async (tables: Table[]): Promise<void> => {
+        for (const table of tables) {
           let filePaths: string[] = [];
 
           if (fileManagerType === 'native' && window.api) {
diff --git a/meerkat-browser-runner/src/app/app.tsx b/meerkat-browser-runner/src/app/app.tsx
@@ -11,7 +11,7 @@ import {
 } from '@devrev/meerkat-dbm';
 
 import log from 'loglevel';
-import { TableWiseFiles } from 'meerkat-dbm/src/types';
+import { Table } from 'meerkat-dbm/src/types';
 import { useEffect, useRef, useState } from 'react';
 import { InstanceManager } from './duck-db/instance-manager';
 
@@ -110,13 +110,13 @@ export function App() {
                 ...message.message.payload,
                 options: {
                   ...message.message.payload.options,
-                  preQuery: async (tableWiseFiles: TableWiseFiles[]) => {
+                  preQuery: async (tables: Table[]) => {
                     const preQueryMessage =
                       await communicationRef.current?.sendRequest<string[]>({
                         type: BROWSER_RUNNER_TYPE.RUNNER_PRE_QUERY,
                         payload: {
                           runnerId: uuid,
-                          tableWiseFiles: tableWiseFiles,
+                          tables: tables,
                         },
                       });
 
diff --git a/meerkat-dbm/package.json b/meerkat-dbm/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@devrev/meerkat-dbm",
-  "version": "0.1.31",
+  "version": "0.1.32",
   "dependencies": {
     "tslib": "^2.3.0",
     "@duckdb/duckdb-wasm": "1.28.1-dev258.0",
diff --git a/meerkat-dbm/src/dbm/__test__/dbm.spec.ts b/meerkat-dbm/src/dbm/__test__/dbm.spec.ts
@@ -4,7 +4,7 @@ import {
   FileJsonStore,
   FileManagerType,
 } from '../../file-manager/file-manager-type';
-import { FileData, Table, TableWiseFiles } from '../../types';
+import { FileData, Table } from '../../types';
 import { DBM } from '../dbm';
 import { DBMConstructorOptions, TableConfig } from '../types';
 import { InstanceManager } from './mock';
@@ -77,10 +77,8 @@ export class MockFileManager implements FileManagerType {
     }
   }
 
-  async getFilesNameForTables(
-    tableNames: TableConfig[]
-  ): Promise<TableWiseFiles[]> {
-    const data: TableWiseFiles[] = [];
+  async getFilesNameForTables(tableNames: TableConfig[]): Promise<Table[]> {
+    const data: Table[] = [];
 
     for (const { name: tableName } of tableNames) {
       const files: FileData[] = [];
diff --git a/meerkat-dbm/src/dbm/dbm-parallel/runner-manager.ts b/meerkat-dbm/src/dbm/dbm-parallel/runner-manager.ts
@@ -1,6 +1,6 @@
 import { FileBufferStore } from '../../file-manager/file-manager-type';
 import { DBMEvent, DBMLogger } from '../../logger';
-import { TableWiseFiles } from '../../types';
+import { Table } from '../../types';
 import {
   BROWSER_RUNNER_TYPE,
   BrowserRunnerMessage,
@@ -13,10 +13,7 @@ export interface IFrameRunnerManagerConstructor {
   runnerURL: string;
   origin: string;
   fetchTableFileBuffers: (tables: TableConfig[]) => Promise<FileBufferStore[]>;
-  fetchPreQuery: (
-    runnerId: string,
-    tableWiseFiles: TableWiseFiles[]
-  ) => string[];
+  fetchPreQuery: (runnerId: string, tables: Table[]) => string[];
   totalRunners: number;
   logger: DBMLogger;
   onEvent?: (event: DBMEvent) => void;
@@ -46,10 +43,7 @@ export class IFrameRunnerManager {
   private fetchTableFileBuffers: (
     tables: TableConfig[]
   ) => Promise<FileBufferStore[]>;
-  private fetchPreQuery: (
-    runnerId: string,
-    tableWiseFiles: TableWiseFiles[]
-  ) => string[];
+  private fetchPreQuery: (runnerId: string, tables: Table[]) => string[];
 
   constructor({
     runnerURL,
@@ -173,9 +167,9 @@ export class IFrameRunnerManager {
             return;
           }
 
-          const { tableWiseFiles } = message.message.payload;
+          const { tables } = message.message.payload;
 
-          const preQueries = this.fetchPreQuery(runnerId, tableWiseFiles);
+          const preQueries = this.fetchPreQuery(runnerId, tables);
 
           manager.communication.sendResponse(message.uuid, preQueries);
         }
diff --git a/meerkat-dbm/src/dbm/types.ts b/meerkat-dbm/src/dbm/types.ts
@@ -1,7 +1,7 @@
 import { AsyncDuckDBConnection } from '@duckdb/duckdb-wasm';
 import { FileManagerType } from '../file-manager/file-manager-type';
 import { DBMEvent, DBMLogger } from '../logger';
-import { TableWiseFiles } from '../types';
+import { Table } from '../types';
 import { InstanceManagerType } from './instance-manager';
 
 export interface DBMConstructorOptions {
@@ -66,9 +66,9 @@ export interface QueryOptions {
   /**
    * @description
    * A callback function which will be executed before the query is executed.
-   * @param tableWiseFiles - An array of tables with associated file names.
+   * @param tables - An array of tables with associated file names.
    */
-  preQuery?: (tableWiseFiles: TableWiseFiles[]) => Promise<void>;
+  preQuery?: (tables: Table[]) => Promise<void>;
 
   /**
    * @description
diff --git a/meerkat-dbm/src/file-manager/__tests__/indexed-db-file-manager.spec.ts b/meerkat-dbm/src/file-manager/__tests__/indexed-db-file-manager.spec.ts
@@ -244,4 +244,30 @@ describe('IndexedDBFileManager', () => {
       expect.arrayContaining(JSON_FILES.map((file) => file.fileName))
     );
   });
+
+  it('should get the files for tables', async () => {
+    const metadata = {
+      test: 'test',
+    };
+
+    await fileManager.registerFileBuffer(fileBuffer);
+
+    await fileManager.setTableMetadata('taxi1', metadata);
+
+    const files = await fileManager.getFilesNameForTables([{ name: 'taxi1' }]);
+
+    expect(files).toEqual([
+      {
+        tableName: 'taxi1',
+        files: [
+          {
+            fileName: 'taxi1.parquet',
+            fileType: 'parquet',
+            partitions: fileBuffer.partitions,
+          },
+        ],
+        metadata,
+      },
+    ]);
+  });
 });
diff --git a/meerkat-dbm/src/file-manager/file-manager-type.ts b/meerkat-dbm/src/file-manager/file-manager-type.ts
@@ -1,7 +1,7 @@
 import { InstanceManagerType } from '../dbm/instance-manager';
 import { TableConfig } from '../dbm/types';
 import { DBMEvent, DBMLogger } from '../logger';
-import { Table, TableWiseFiles } from '../types';
+import { Table } from '../types';
 
 export interface FileManagerConstructorOptions {
   /**
@@ -128,7 +128,7 @@ export interface FileManagerType {
    * @param tables - An array of tables.
    * @returns Array of objects containing table names and associated files.
    */
-  getFilesNameForTables: (tables: TableConfig[]) => Promise<TableWiseFiles[]>;
+  getFilesNameForTables: (tables: TableConfig[]) => Promise<Table[]>;
 
   /**
    * @description
diff --git a/meerkat-dbm/src/file-manager/indexed-db/base-indexed-db-file-manager.ts b/meerkat-dbm/src/file-manager/indexed-db/base-indexed-db-file-manager.ts
@@ -1,7 +1,7 @@
 import { InstanceManagerType } from '../../dbm/instance-manager';
 import { TableConfig } from '../../dbm/types';
 import { DBMEvent, DBMLogger } from '../../logger';
-import { Table, TableWiseFiles } from '../../types';
+import { Table } from '../../types';
 import { getBufferFromJSON, isDefined } from '../../utils';
 import {
   FileBufferStore,
@@ -77,25 +77,16 @@ export abstract class BaseIndexedDBFileManager implements FileManagerType {
     });
   }
 
-  async getFilesNameForTables(
-    tables: TableConfig[]
-  ): Promise<TableWiseFiles[]> {
-    const tableNames = tables.map((table) => table.name);
-
-    const tableData = (await this.indexedDB.tablesKey.bulkGet(tableNames))
-      .filter(isDefined)
-      .reduce((tableObj, table) => {
-        tableObj[table.tableName] = table;
-        return tableObj;
-      }, {} as { [key: string]: Table });
-
-    return tables.map((table) => ({
-      tableName: table.name,
-      files: getFilesByPartition(
-        tableData[table.name]?.files ?? [],
-        table.partitions
-      ),
-    }));
+  async getFilesNameForTables(tables: TableConfig[]): Promise<Table[]> {
+    const results = await Promise.all(
+      tables.map(async (table) => {
+        const tableData = await this.getTableData(table);
+
+        return tableData;
+      })
+    );
+
+    return results.filter(isDefined);
   }
 
   async getTableData(table: TableConfig): Promise<Table | undefined> {
diff --git a/meerkat-dbm/src/file-manager/memory/memory-file-manager.ts b/meerkat-dbm/src/file-manager/memory/memory-file-manager.ts
@@ -1,7 +1,7 @@
 import { InstanceManagerType } from '../../dbm/instance-manager';
 import { TableConfig } from '../../dbm/types';
 import { DBMEvent, DBMLogger } from '../../logger';
-import { Table, TableWiseFiles } from '../../types';
+import { Table } from '../../types';
 import { getBufferFromJSON } from '../../utils';
 import {
   FileBufferStore,
@@ -82,9 +82,7 @@ export class MemoryDBFileManager implements FileManagerType {
     // not needed for memory file manager
   }
 
-  async getFilesNameForTables(
-    tables: TableConfig[]
-  ): Promise<TableWiseFiles[]> {
+  async getFilesNameForTables(tables: TableConfig[]): Promise<Table[]> {
     // not needed for memory file manager
     return [];
   }
diff --git a/meerkat-dbm/src/file-manager/memory/parallel-memory-file-manager.ts b/meerkat-dbm/src/file-manager/memory/parallel-memory-file-manager.ts
@@ -1,7 +1,7 @@
 import { InstanceManagerType } from '../../dbm/instance-manager';
 import { TableConfig } from '../../dbm/types';
 import { DBMEvent, DBMLogger } from '../../logger';
-import { Table, TableWiseFiles } from '../../types';
+import { Table } from '../../types';
 import {
   convertUint8ArrayToSharedArrayBuffer,
   getBufferFromJSON,
@@ -115,9 +115,7 @@ export class ParallelMemoryFileManager
     // no-op
   }
 
-  async getFilesNameForTables(
-    tables: TableConfig[]
-  ): Promise<TableWiseFiles[]> {
+  async getFilesNameForTables(tables: TableConfig[]): Promise<Table[]> {
     return [];
   }
 
diff --git a/meerkat-dbm/src/file-manager/memory/runner-memory-file-manager.ts b/meerkat-dbm/src/file-manager/memory/runner-memory-file-manager.ts
@@ -1,7 +1,7 @@
 import { InstanceManagerType } from '../../dbm/instance-manager';
 import { TableConfig } from '../../dbm/types';
 import { DBMEvent, DBMLogger } from '../../logger';
-import { Table, TableWiseFiles } from '../../types';
+import { Table } from '../../types';
 import {
   convertSharedArrayBufferToUint8Array,
   getBufferFromJSON,
@@ -152,9 +152,7 @@ export class RunnerMemoryDBFileManager implements FileManagerType {
     });
   }
 
-  async getFilesNameForTables(
-    tables: TableConfig[]
-  ): Promise<TableWiseFiles[]> {
+  async getFilesNameForTables(tables: TableConfig[]): Promise<Table[]> {
     const tableNames = tables.map((table) => table.name);
 
     return tableNames.map((tableName) => {
diff --git a/meerkat-dbm/src/index.ts b/meerkat-dbm/src/index.ts
@@ -1,6 +1,7 @@
 export * from './dbm';
 export * from './file-manager';
 export * from './logger';
+export * from './types';
 export {
   convertArrowTableToJSON,
   convertSharedArrayBufferToUint8Array,
diff --git a/meerkat-dbm/src/types/common-types.ts b/meerkat-dbm/src/types/common-types.ts
diff --git a/meerkat-dbm/src/types/index.ts b/meerkat-dbm/src/types/index.ts
@@ -1,3 +1 @@
-export * from './common-types';
 export * from './db-schema';
-
diff --git a/meerkat-dbm/src/window-communication/runner-types.ts b/meerkat-dbm/src/window-communication/runner-types.ts
@@ -1,6 +1,6 @@
 import { QueryOptions, TableConfig } from '../dbm/types';
 import { DBMEvent } from '../logger';
-import { TableWiseFiles } from '../types';
+import { Table } from '../types';
 
 export const BROWSER_RUNNER_TYPE = {
   RUNNER_ON_READY: 'RUNNER_ON_READY',
@@ -47,7 +47,7 @@ export interface BrowserRunnerGetFileBuffersMessage {
 export interface BrowserRunnerPreQueryMessage {
   type: typeof BROWSER_RUNNER_TYPE.RUNNER_PRE_QUERY;
   payload: {
-    tableWiseFiles: TableWiseFiles[];
+    tables: Table[];
   };
 }
 

Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"name": "@devrev/meerkat-dbm",`
`3`		`- "version": "0.1.31",`
	`3`	`+ "version": "0.1.32",`
`4`	`4`	`"dependencies": {`
`5`	`5`	`"tslib": "^2.3.0",`
`6`	`6`	`"@duckdb/duckdb-wasm": "1.28.1-dev258.0",`