feat: add support for json register (#47)

* add: json method

* add: helper

* update: path

* add: tests

* rever: prettier change

* add: events and logger

* update: comments

* add: json in metadata

* remove: params

* add: comments

* resolve: comment

* change: method

* nit: changes

* add: tests

* fix: test

* address: comments

* remove: type

* revert: settings

* update: test

* change: json

Author: vpbs2

benchmarking/benchmarking-app/src/app/constants.ts

@@ -0,0 +1,38 @@
+export const TEST_QUERIES = [
+  'SELECT CAST(COUNT(*) as VARCHAR) as total_count FROM taxi.parquet',
+  "SELECT * FROM taxi.parquet WHERE originating_base_num='B03404' LIMIT 100",
+  'SELECT CAST(COUNT(*) as VARCHAR) as total_count FROM taxi.parquet GROUP BY hvfhs_license_num',
+  'SELECT * as total_count FROM taxi.parquet ORDER BY bcf LIMIT 100',
+  `
+      WITH group_by_query AS (
+        SELECT
+            hvfhs_license_num,
+            COUNT(*)
+        FROM
+            taxi.parquet
+        GROUP BY
+            hvfhs_license_num
+    ),
+
+    full_query AS (
+        SELECT
+            *
+        FROM
+            taxi.parquet
+    )
+
+    SELECT
+        COUNT(*)
+    FROM
+        group_by_query
+    LEFT JOIN
+        full_query
+    ON
+        group_by_query.hvfhs_license_num = full_query.hvfhs_license_num
+    LIMIT 1
+      `,
+  'SELECT CAST(COUNT(*) as VARCHAR) as total_count FROM taxijson',
+  'SELECT * FROM taxijson WHERE price >= 1.0005812645 LIMIT 100',
+  'SELECT CAST(COUNT(*) as VARCHAR) as total_count FROM taxijson GROUP BY order_count',
+  'SELECT * as total_count FROM taxijson ORDER BY seconds_in_bucket LIMIT 100',
+];

benchmarking/benchmarking-app/src/app/dbm-context/memory-dbm-context.tsx

@@ -24,6 +24,10 @@ export const MemoryDBMProvider = ({ children }: { children: JSX.Element }) => {
       fetchTableFileBuffers: async (table) => {
         return [];
       },
+      logger: log,
+      onEvent: (event) => {
+        console.info(event);
+      },
     });
     log.setLevel('DEBUG');
     const dbm = new DBM({

benchmarking/benchmarking-app/src/app/file-loader/file-loader.tsx

@@ -1,5 +1,6 @@
 import axios from 'axios';
 import { useState } from 'react';
+import TAXI_JSON_DATA from '../../assets/data-sets/taxi.json';
 import { useDBM } from '../hooks/dbm-context';
 import { useClassicEffect } from '../hooks/use-classic-effect';
 
@@ -21,6 +22,12 @@ export const FileLoader = ({ children }: { children: JSX.Element }) => {
         buffer: fileBufferView,
       });
 
+      await fileManager.registerJSON({
+        json: TAXI_JSON_DATA,
+        tableName: 'taxijson',
+        fileName: 'taxi.json',
+      });
+
       setIsFileLoader(true);
     })();
   }, []);

benchmarking/benchmarking-app/src/app/query-benchmarking/query-benchmarking.tsx

@@ -1,4 +1,5 @@
 import { useState } from 'react';
+import { TEST_QUERIES } from '../constants';
 import { useDBM } from '../hooks/dbm-context';
 import { useClassicEffect } from '../hooks/use-classic-effect';
 
@@ -14,49 +15,15 @@ export const QueryBenchmarking = () => {
 
   useClassicEffect(() => {
     setTotalTime(0);
-    const testQueries = [
-      'SELECT CAST(COUNT(*) as VARCHAR) as total_count FROM taxi.parquet',
-      "SELECT * FROM taxi.parquet WHERE originating_base_num='B03404' LIMIT 100",
-      'SELECT CAST(COUNT(*) as VARCHAR) as total_count FROM taxi.parquet GROUP BY hvfhs_license_num',
-      'SELECT * as total_count FROM taxi.parquet ORDER BY bcf LIMIT 100',
-      `
-      WITH group_by_query AS (
-        SELECT
-            hvfhs_license_num,
-            COUNT(*)
-        FROM
-            taxi.parquet
-        GROUP BY
-            hvfhs_license_num
-    ),
-
-    full_query AS (
-        SELECT
-            *
-        FROM
-            taxi.parquet
-    )
-
-    SELECT
-        COUNT(*)
-    FROM
-        group_by_query
-    LEFT JOIN
-        full_query
-    ON
-        group_by_query.hvfhs_license_num = full_query.hvfhs_license_num
-    LIMIT 1
-      `,
-    ];
 
     setOutput([]);
     const promiseArr = [];
     const start = performance.now();
-    for (let i = 0; i < testQueries.length; i++) {
+    for (let i = 0; i < TEST_QUERIES.length; i++) {
       const eachQueryStart = performance.now();
 
       const promiseObj = dbm
-        .queryWithTableNames(testQueries[i], ['taxi'])
+        .queryWithTableNames(TEST_QUERIES[i], ['taxi'])
         .then((results) => {
           const end = performance.now();
           const time = end - eachQueryStart;

benchmarking/benchmarking-app/src/assets/data-sets/taxi.json

@@ -4,7 +4,8 @@
     "allowJs": false,
     "esModuleInterop": false,
     "allowSyntheticDefaultImports": true,
-    "strict": true
+    "strict": true,
+    "resolveJsonModule": true,
   },
   "files": [],
   "include": [],

benchmarking/benchmarking-app/tsconfig.json

@@ -1,6 +1,6 @@
 {
   "name": "@devrev/meerkat-dbm",
-  "version": "0.0.136",
+  "version": "0.0.14",
   "dependencies": {
     "tslib": "^2.3.0",
     "@duckdb/duckdb-wasm": "^1.28.0",

meerkat-dbm/package.json

@@ -2,6 +2,7 @@ import { AsyncDuckDB } from '@duckdb/duckdb-wasm';
 import log from 'loglevel';
 import {
   FileBufferStore,
+  FileJsonStore,
   FileManagerType,
 } from '../file-manager/file-manager-type';
 import { FileData, Table, TableWiseFiles } from '../types';
@@ -28,6 +29,15 @@ export class MockFileManager implements FileManagerType {
     this.tables[prop.tableName].files.push(prop);
   }
 
+  async registerJSON(prop: FileJsonStore): Promise<void> {
+    const { json, ...fileData } = prop;
+
+    this.registerFileBuffer({
+      ...fileData,
+      buffer: new Uint8Array(),
+    });
+  }
+
   async getFileBuffer(name: string): Promise<Uint8Array> {
     const fileBuffer = this.fileBufferStore[name];
     if (!fileBuffer) {

meerkat-dbm/src/dbm/dbm.spec.ts

@@ -1,7 +1,6 @@
 import { AsyncDuckDBConnection } from '@duckdb/duckdb-wasm';
 import { FileManagerType } from '../file-manager/file-manager-type';
-import { DBMEvent } from '../logger/event-types';
-import { DBMLogger } from '../logger/logger-types';
+import { DBMEvent, DBMLogger } from '../logger';
 import { InstanceManagerType } from './instance-manager';
 
 import { DBMConstructorOptions, QueryOptions, QueryQueueItem } from './types';

meerkat-dbm/src/dbm/dbm.ts

@@ -1,6 +1,5 @@
 import { FileManagerType } from '../file-manager/file-manager-type';
-import { DBMEvent } from '../logger/event-types';
-import { DBMLogger } from '../logger/logger-types';
+import { DBMEvent, DBMLogger } from '../logger';
 import { TableWiseFiles } from '../types';
 import { InstanceManagerType } from './instance-manager';