feat(filters): added support for set and not set in where clause

meerkat-browser/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@devrev/meerkat-browser",
-  "version": "0.0.71",
+  "version": "0.0.72",
   "dependencies": {
     "@swc/helpers": "~0.5.0",
     "@devrev/meerkat-core": "*",

meerkat-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@devrev/meerkat-core",
-  "version": "0.0.73",
+  "version": "0.0.74",
   "dependencies": {
     "@swc/helpers": "~0.5.0"
   },

meerkat-core/src/cube-filter-transformer/factory.ts

@@ -25,7 +25,9 @@ import { lteTransform } from './lte/lte';
 import { notInDataRangeTransform } from './not-In-date-range/not-In-date-range';
 import { notContainsTransform } from './not-contains/not-contains';
 import { notEqualsTransform } from './not-equals/not-equals';
+import { notSetTransform } from './not-set/not-set';
 import { orDuckdbCondition } from './or/or';
+import { setTransform } from './set/set';
 
 export type CubeToParseExpressionTransform = (
   query: QueryOperatorsWithInfo
@@ -54,6 +56,12 @@ const cubeFilterOperatorsToDuckdb = (cubeFilter: QueryOperatorsWithInfo) => {
       return inDataRangeTransform(cubeFilter);
     case 'notInDateRange':
       return notInDataRangeTransform(cubeFilter);
+    case 'notSet': {
+      return notSetTransform(cubeFilter);
+    }
+    case 'set': {
+      return setTransform(cubeFilter);
+    }
     default:
       throw new Error('Could not transform the filter');
   }

meerkat-core/src/cube-filter-transformer/not-set/not-set.spec.ts

@@ -0,0 +1,50 @@
+import { notSetTransform } from "./not-set";
+
+describe("notSetTransform", () => {
+    it("should return the correct expression for a given query", () => {
+        const query = {
+            member: "table.column"
+        };
+
+        const expectedExpression = {
+            class: "OPERATOR",
+            type: "OPERATOR_IS_NULL",
+            alias: "",
+            children: [
+                {
+                    class: "COLUMN_REF",
+                    type: "COLUMN_REF",
+                    alias: "",
+                    column_names: ["table", "column"]
+                }
+            ]
+        };
+
+        const result = notSetTransform(query);
+
+        expect(result).toEqual(expectedExpression);
+    });
+    it("should return the correct expression for a __ delimited query", () => {
+        const query = {
+            member: "table__column"
+        };
+
+        const expectedExpression = {
+            class: "OPERATOR",
+            type: "OPERATOR_IS_NULL",
+            alias: "",
+            children: [
+                {
+                    class: "COLUMN_REF",
+                    type: "COLUMN_REF",
+                    alias: "",
+                    column_names: ["table__column"]
+                }
+            ]
+        };
+
+        const result = notSetTransform(query);
+
+        expect(result).toEqual(expectedExpression);
+    });
+});

meerkat-core/src/cube-filter-transformer/not-set/not-set.ts

@@ -0,0 +1,19 @@
+import { ExpressionClass, ExpressionType } from '../../types/duckdb-serialization-types/serialization/Expression';
+import { CubeToParseExpressionTransform } from "../factory";
+
+export const notSetTransform: CubeToParseExpressionTransform = (query)  => {
+    const { member } = query;
+    return {
+        class: ExpressionClass.OPERATOR,
+        type: ExpressionType.OPERATOR_IS_NULL,
+        alias: "",
+        children: [
+            {
+                class: "COLUMN_REF",
+                type: "COLUMN_REF",
+                alias: "",
+                column_names: member.split('.')
+            }
+        ]
+    }
+}

meerkat-core/src/cube-filter-transformer/set/set.spec.ts

@@ -0,0 +1,51 @@
+import { setTransform } from './set';
+
+describe('setTransform', () => {
+    it('should return the correct expression object', () => {
+        const query = {
+            member: 'table.column'
+        };
+
+        const expected = {
+            class: 'OPERATOR',
+            type: 'OPERATOR_IS_NOT_NULL',
+            alias: '',
+            children: [
+                {
+                    class: 'COLUMN_REF',
+                    type: 'COLUMN_REF',
+                    alias: '',
+                    column_names: ['table', 'column']
+                }
+            ]
+        };
+
+        const result = setTransform(query);
+
+        expect(result).toEqual(expected);
+    });
+
+    it('should handle __ delimited query', () => {
+        const query = {
+            member: 'table__column'
+        };
+
+        const expected = {
+            class: 'OPERATOR',
+            type: 'OPERATOR_IS_NOT_NULL',
+            alias: '',
+            children: [
+                {
+                    class: 'COLUMN_REF',
+                    type: 'COLUMN_REF',
+                    alias: '',
+                    column_names: ['table__column']
+                }
+            ]
+        };
+
+        const result = setTransform(query);
+
+        expect(result).toEqual(expected);
+    });
+});

meerkat-core/src/cube-filter-transformer/set/set.ts

@@ -0,0 +1,19 @@
+import { ExpressionClass, ExpressionType } from '../../types/duckdb-serialization-types/serialization/Expression';
+import { CubeToParseExpressionTransform } from "../factory";
+
+export const setTransform: CubeToParseExpressionTransform = (query)  => {
+    const { member } = query;
+    return {
+        class: ExpressionClass.OPERATOR,
+        type: ExpressionType.OPERATOR_IS_NOT_NULL,
+        alias: "",
+        children: [
+            {
+                class: "COLUMN_REF",
+                type: "COLUMN_REF",
+                alias: "",
+                column_names: member.split('.')
+            }
+        ]
+    }
+}

meerkat-core/src/utils/key-from-measures-dimension.ts

@@ -7,7 +7,6 @@ export const getMemberInfoFromTableSchema = (
   let memberInfo: Measure | Dimension | undefined;
 
   const memberKeyName = memberKey.split('.')[1];
-  console.info('memberKeyName', memberKeyName, memberKey);
 
   /**
    * Finding the table key from the measures.

meerkat-node/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@devrev/meerkat-node",
-  "version": "0.0.71",
+  "version": "0.0.72",
   "dependencies": {
     "@swc/helpers": "~0.5.0",
     "@devrev/meerkat-core": "*",

meerkat-node/src/__tests__/cube-filter-params.spec.ts

@@ -42,7 +42,8 @@ describe('filter-param-tests', () => {
              (4, DATE '2022-03-01', 'cancelled', 40.00),
              (5, DATE '2022-01-28', 'completed', 80.75),
              (6, DATE '2022-02-15', 'pending', 120.00),
-             (7, DATE '2022-04-01', 'completed', 210.00);`);
+             (7, DATE '2022-02-15', 'pending', 120.00),
+             (8, DATE '2022-04-01', 'initiated', null);`);
   });
 
   it('Should apply filter params to base SQL', async () => {
@@ -60,7 +61,7 @@ describe('filter-param-tests', () => {
     const sql = await cubeQueryToSQL(query, [SCHEMA]);
     console.info('SQL: ', sql);
     const output: any = await duckdbExec(sql);
-    expect(output).toHaveLength(1);
+    expect(output).toHaveLength(2);
     expect(output[0].id).toBe(6);
   });
 
@@ -93,7 +94,7 @@ describe('filter-param-tests', () => {
     const sql = await cubeQueryToSQL(query, [SCHEMA]);
     console.info('SQL: ', sql);
     const output: any = await duckdbExec(sql);
-    expect(output).toHaveLength(2);
+    expect(output).toHaveLength(3);
     expect(output[0].id).toBe(6);
   });
 
@@ -107,7 +108,7 @@ describe('filter-param-tests', () => {
     const sql = await cubeQueryToSQL(query, [SCHEMA]);
     console.info('SQL: ', sql);
     const output: any = await duckdbExec(sql);
-    expect(output).toHaveLength(7);
+    expect(output).toHaveLength(8);
   });
 
   it('Should apply true filter if filters are present but are not matching', async () => {
@@ -139,6 +140,31 @@ describe('filter-param-tests', () => {
     const sql = await cubeQueryToSQL(query, [SCHEMA]);
     console.info('SQL: ', sql);
     const output: any = await duckdbExec(sql);
-    expect(output).toHaveLength(4);
+    expect(output).toHaveLength(5);
+  });
+  it('Should apply notSet and set filters', async () => {
+    const query = {
+      measures: ['*'],
+      filters: [
+        {
+          and: [
+            {
+              member: 'orders.amount',
+              operator: 'notSet',
+            },
+            {
+              member: 'orders.status',
+              operator: 'set',
+            }
+          ],
+        }, 
+      ],
+      dimensions: [],
+    };
+
+    const sql = await cubeQueryToSQL(query, [SCHEMA]);
+    console.info('SQL: ', sql);
+    const output: any = await duckdbExec(sql);
+    expect(output).toHaveLength(1);
   });
 });