feat: dimension validator (#118)

* add: dimension validator

* some cleanup

* update: structure

* add: tests

* remove: testing

* update: test

* add: more test

* fix: build

Author: vpbs2

meerkat-browser/src/ast/dimension.ts

@@ -0,0 +1,32 @@
+import { validateDimension } from '@devrev/meerkat-core';
+import { AsyncDuckDBConnection } from '@duckdb/duckdb-wasm';
+import { parseQueryToAST } from './query-to-ast';
+import { getAvailableFunctions, isParseError } from './utils';
+
+/**
+ * Validates the query can be used as a dimension by parsing it to an AST and checking its structure
+ * @param connection - DuckDB connection instance
+ * @param query - The query string to validate
+ * @returns Promise<boolean> - Whether the dimension is valid
+ */
+export const validateDimensionQuery = async ({
+  connection,
+  query,
+  validFunctions,
+}: {
+  connection: AsyncDuckDBConnection;
+  query: string;
+  validFunctions?: string[];
+}): Promise<boolean> => {
+  const parsedSerialization = await parseQueryToAST(query, connection);
+
+  if (isParseError(parsedSerialization)) {
+    throw new Error(parsedSerialization.error_message ?? 'Unknown error');
+  }
+
+  // Only fetch valid functions if not provided
+  const availableFunctions =
+    validFunctions ?? (await getAvailableFunctions(connection, 'scalar'));
+
+  return validateDimension(parsedSerialization, availableFunctions);
+};

meerkat-browser/src/ast/index.ts

@@ -0,0 +1,2 @@
+export * from './dimension';
+export * from './query-to-ast';

meerkat-browser/src/ast/query-to-ast.ts

@@ -0,0 +1,29 @@
+import {
+  astSerializerQuery,
+  deserializeQuery,
+  ParsedSerialization,
+} from '@devrev/meerkat-core';
+import { AsyncDuckDBConnection } from '@duckdb/duckdb-wasm';
+
+/**
+ * Converts a query to an AST
+ * @param query - The query string to convert
+ * @param connection - The DuckDB connection instance
+ * @returns The AST as a JSON object
+ */
+export const parseQueryToAST = async (
+  query: string,
+  connection: AsyncDuckDBConnection
+): Promise<ParsedSerialization> => {
+  try {
+    const serializedQuery = astSerializerQuery(query);
+    const arrowResult = await connection.query(serializedQuery);
+
+    const parsedOutputQuery = arrowResult.toArray().map((row) => row.toJSON());
+    const deserializedQuery = deserializeQuery(parsedOutputQuery);
+
+    return JSON.parse(deserializedQuery);
+  } catch (error) {
+    throw new Error('Failed to parse query to AST');
+  }
+};

meerkat-browser/src/ast/utils.ts

@@ -0,0 +1,18 @@
+import { ParsedSerialization } from '@devrev/meerkat-core';
+import { AsyncDuckDBConnection } from '@duckdb/duckdb-wasm';
+
+export const isParseError = (data: ParsedSerialization): boolean => {
+  return !!data.error;
+};
+
+// Helper function to get available functions from DuckDB based on function type
+export const getAvailableFunctions = async (
+  connection: AsyncDuckDBConnection,
+  function_type: 'scalar' | 'aggregate'
+): Promise<string[]> => {
+  const result = await connection.query(
+    `SELECT distinct function_name FROM duckdb_functions() WHERE function_type = '${function_type}'`
+  );
+
+  return result.toArray().map((row) => row.toJSON().function_name);
+};

meerkat-browser/src/index.ts

@@ -1,3 +1,4 @@
 export * from './browser-cube-to-sql/browser-cube-to-sql';
 export { convertCubeStringToTableSchema };
 import { convertCubeStringToTableSchema } from '@devrev/meerkat-core';
+export { validateDimensionQuery } from './ast';

meerkat-core/src/ast-serializer/ast-serializer.ts

@@ -0,0 +1,3 @@
+export const astSerializerQuery = (query: string) => {
+  return `SELECT json_serialize_sql('${query}')`;
+};