Skip to content

Commit 27f8bc3

Browse files
vpbs2vpbs2
committed
refactor prequery
1 parent 636add9 commit 27f8bc3

File tree

17 files changed

+50
-80
lines changed

17 files changed

+50
-80
lines changed

‎benchmarking/src/app/dbm-context/parallel-indexed-dbm-context.tsx

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,9 @@ import {
22
DBMParallel,
33
IFrameRunnerManager,
44
ParallelIndexedDBFileManager,
5+
Table,
56
} from '@devrev/meerkat-dbm';
67
import log from 'loglevel';
7-
import { TableWiseFiles } from 'meerkat-dbm/src/types/common-types';
88
import { useRef, useState } from 'react';
99
import { DBMContext } from '../hooks/dbm-context';
1010
import { useClassicEffect } from '../hooks/use-classic-effect';
@@ -45,14 +45,14 @@ export const ParallelIndexedDBMProvider = ({
4545
fetchTableFileBuffers: async (table) => {
4646
return [];
4747
},
48-
fetchPreQuery: (runnerId: string, tableWiseFiles: TableWiseFiles[]) => {
48+
fetchPreQuery: (runnerId: string, tables: Table[]) => {
4949
const preQueries: string[] = [];
5050

51-
for (const tableWiseFile of tableWiseFiles) {
51+
for (const tableData of tables) {
5252
preQueries.push(
5353
generateViewQuery(
54-
tableWiseFile.tableName,
55-
tableWiseFile.files.map((file) => file.fileName)
54+
tableData.tableName,
55+
tableData.files.map((file) => file.fileName)
5656
)
5757
);
5858
}

‎benchmarking/src/app/dbm-context/parallel-memory-dbm-context.tsx

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,9 @@ import {
22
DBMParallel,
33
IFrameRunnerManager,
44
ParallelMemoryFileManager,
5+
Table,
56
} from '@devrev/meerkat-dbm';
67
import log from 'loglevel';
7-
import { TableWiseFiles } from 'meerkat-dbm/src/types/common-types';
88
import { useRef, useState } from 'react';
99
import { DBMContext } from '../hooks/dbm-context';
1010
import { useClassicEffect } from '../hooks/use-classic-effect';
@@ -45,14 +45,14 @@ export const ParallelMemoryDBMProvider = ({
4545
fetchTableFileBuffers: async (table) => {
4646
return fileManagerRef.current.getTableBufferData(table);
4747
},
48-
fetchPreQuery: (runnerId: string, tableWiseFiles: TableWiseFiles[]) => {
48+
fetchPreQuery: (runnerId: string, tables: Table[]) => {
4949
const preQueries: string[] = [];
5050

51-
for (const tableWiseFile of tableWiseFiles) {
51+
for (const tableData of tables) {
5252
preQueries.push(
5353
generateViewQuery(
54-
tableWiseFile.tableName,
55-
tableWiseFile.files.map((file) => file.fileName)
54+
tableData.tableName,
55+
tableData.files.map((file) => file.fileName)
5656
)
5757
);
5858
}

‎benchmarking/src/app/query-benchmarking/query-benchmarking.tsx

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { TableWiseFiles } from 'meerkat-dbm/src/types';
1+
import { Table } from 'meerkat-dbm/src/types';
22
import { useMemo, useState } from 'react';
33
import { TEST_QUERIES } from '../constants';
44
import { useDBM } from '../hooks/dbm-context';
@@ -17,8 +17,8 @@ export const QueryBenchmarking = () => {
1717

1818
const preQuery = useMemo(
1919
() =>
20-
async (tablesFileData: TableWiseFiles[]): Promise<void> => {
21-
for (const table of tablesFileData) {
20+
async (tables: Table[]): Promise<void> => {
21+
for (const table of tables) {
2222
let filePaths: string[] = [];
2323

2424
if (fileManagerType === 'native' && window.api) {

‎meerkat-browser-runner/src/app/app.tsx

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ import {
1111
} from '@devrev/meerkat-dbm';
1212

1313
import log from 'loglevel';
14-
import { TableWiseFiles } from 'meerkat-dbm/src/types';
14+
import { Table } from 'meerkat-dbm/src/types';
1515
import { useEffect, useRef, useState } from 'react';
1616
import { InstanceManager } from './duck-db/instance-manager';
1717

@@ -110,13 +110,13 @@ export function App() {
110110
...message.message.payload,
111111
options: {
112112
...message.message.payload.options,
113-
preQuery: async (tableWiseFiles: TableWiseFiles[]) => {
113+
preQuery: async (tables: Table[]) => {
114114
const preQueryMessage =
115115
await communicationRef.current?.sendRequest<string[]>({
116116
type: BROWSER_RUNNER_TYPE.RUNNER_PRE_QUERY,
117117
payload: {
118118
runnerId: uuid,
119-
tableWiseFiles: tableWiseFiles,
119+
tables: tables,
120120
},
121121
});
122122

‎meerkat-dbm/package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@devrev/meerkat-dbm",
3-
"version": "0.1.31",
3+
"version": "0.1.32",
44
"dependencies": {
55
"tslib": "^2.3.0",
66
"@duckdb/duckdb-wasm": "1.28.1-dev258.0",

‎meerkat-dbm/src/dbm/__test__/dbm.spec.ts

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import {
44
FileJsonStore,
55
FileManagerType,
66
} from '../../file-manager/file-manager-type';
7-
import { FileData, Table, TableWiseFiles } from '../../types';
7+
import { FileData, Table } from '../../types';
88
import { DBM } from '../dbm';
99
import { DBMConstructorOptions, TableConfig } from '../types';
1010
import { InstanceManager } from './mock';
@@ -77,10 +77,8 @@ export class MockFileManager implements FileManagerType {
7777
}
7878
}
7979

80-
async getFilesNameForTables(
81-
tableNames: TableConfig[]
82-
): Promise<TableWiseFiles[]> {
83-
const data: TableWiseFiles[] = [];
80+
async getFilesNameForTables(tableNames: TableConfig[]): Promise<Table[]> {
81+
const data: Table[] = [];
8482

8583
for (const { name: tableName } of tableNames) {
8684
const files: FileData[] = [];

‎meerkat-dbm/src/dbm/dbm-parallel/runner-manager.ts

Lines changed: 5 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import { FileBufferStore } from '../../file-manager/file-manager-type';
22
import { DBMEvent, DBMLogger } from '../../logger';
3-
import { TableWiseFiles } from '../../types';
3+
import { Table } from '../../types';
44
import {
55
BROWSER_RUNNER_TYPE,
66
BrowserRunnerMessage,
@@ -13,10 +13,7 @@ export interface IFrameRunnerManagerConstructor {
1313
runnerURL: string;
1414
origin: string;
1515
fetchTableFileBuffers: (tables: TableConfig[]) => Promise<FileBufferStore[]>;
16-
fetchPreQuery: (
17-
runnerId: string,
18-
tableWiseFiles: TableWiseFiles[]
19-
) => string[];
16+
fetchPreQuery: (runnerId: string, tables: Table[]) => string[];
2017
totalRunners: number;
2118
logger: DBMLogger;
2219
onEvent?: (event: DBMEvent) => void;
@@ -46,10 +43,7 @@ export class IFrameRunnerManager {
4643
private fetchTableFileBuffers: (
4744
tables: TableConfig[]
4845
) => Promise<FileBufferStore[]>;
49-
private fetchPreQuery: (
50-
runnerId: string,
51-
tableWiseFiles: TableWiseFiles[]
52-
) => string[];
46+
private fetchPreQuery: (runnerId: string, tables: Table[]) => string[];
5347

5448
constructor({
5549
runnerURL,
@@ -173,9 +167,9 @@ export class IFrameRunnerManager {
173167
return;
174168
}
175169

176-
const { tableWiseFiles } = message.message.payload;
170+
const { tables } = message.message.payload;
177171

178-
const preQueries = this.fetchPreQuery(runnerId, tableWiseFiles);
172+
const preQueries = this.fetchPreQuery(runnerId, tables);
179173

180174
manager.communication.sendResponse(message.uuid, preQueries);
181175
}

‎meerkat-dbm/src/dbm/types.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { AsyncDuckDBConnection } from '@duckdb/duckdb-wasm';
22
import { FileManagerType } from '../file-manager/file-manager-type';
33
import { DBMEvent, DBMLogger } from '../logger';
4-
import { TableWiseFiles } from '../types';
4+
import { Table } from '../types';
55
import { InstanceManagerType } from './instance-manager';
66

77
export interface DBMConstructorOptions {
@@ -66,9 +66,9 @@ export interface QueryOptions {
6666
/**
6767
* @description
6868
* A callback function which will be executed before the query is executed.
69-
* @param tableWiseFiles - An array of tables with associated file names.
69+
* @param tables - An array of tables with associated file names.
7070
*/
71-
preQuery?: (tableWiseFiles: TableWiseFiles[]) => Promise<void>;
71+
preQuery?: (tables: Table[]) => Promise<void>;
7272

7373
/**
7474
* @description

‎meerkat-dbm/src/file-manager/file-manager-type.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { InstanceManagerType } from '../dbm/instance-manager';
22
import { TableConfig } from '../dbm/types';
33
import { DBMEvent, DBMLogger } from '../logger';
4-
import { Table, TableWiseFiles } from '../types';
4+
import { Table } from '../types';
55

66
export interface FileManagerConstructorOptions {
77
/**
@@ -128,7 +128,7 @@ export interface FileManagerType {
128128
* @param tables - An array of tables.
129129
* @returns Array of objects containing table names and associated files.
130130
*/
131-
getFilesNameForTables: (tables: TableConfig[]) => Promise<TableWiseFiles[]>;
131+
getFilesNameForTables: (tables: TableConfig[]) => Promise<Table[]>;
132132

133133
/**
134134
* @description

‎meerkat-dbm/src/file-manager/indexed-db/base-indexed-db-file-manager.ts

Lines changed: 11 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { InstanceManagerType } from '../../dbm/instance-manager';
22
import { TableConfig } from '../../dbm/types';
33
import { DBMEvent, DBMLogger } from '../../logger';
4-
import { Table, TableWiseFiles } from '../../types';
4+
import { Table } from '../../types';
55
import { getBufferFromJSON, isDefined } from '../../utils';
66
import {
77
FileBufferStore,
@@ -77,25 +77,16 @@ export abstract class BaseIndexedDBFileManager implements FileManagerType {
7777
});
7878
}
7979

80-
async getFilesNameForTables(
81-
tables: TableConfig[]
82-
): Promise<TableWiseFiles[]> {
83-
const tableNames = tables.map((table) => table.name);
84-
85-
const tableData = (await this.indexedDB.tablesKey.bulkGet(tableNames))
86-
.filter(isDefined)
87-
.reduce((tableObj, table) => {
88-
tableObj[table.tableName] = table;
89-
return tableObj;
90-
}, {} as { [key: string]: Table });
91-
92-
return tables.map((table) => ({
93-
tableName: table.name,
94-
files: getFilesByPartition(
95-
tableData[table.name]?.files ?? [],
96-
table.partitions
97-
),
98-
}));
80+
async getFilesNameForTables(tables: TableConfig[]): Promise<Table[]> {
81+
const results = await Promise.all(
82+
tables.map(async (table) => {
83+
const tableData = await this.getTableData(table);
84+
85+
return tableData;
86+
})
87+
);
88+
89+
return results.filter(isDefined);
9990
}
10091

10192
async getTableData(table: TableConfig): Promise<Table | undefined> {

0 commit comments

Comments
 (0)