From cddbcaf14bd1dd472f4ca7d989d0c8e7fad30c91 Mon Sep 17 00:00:00 2001 From: zaidjan Date: Sun, 28 Apr 2024 23:31:40 +0530 Subject: [PATCH 1/5] added support for set and not set in where clause --- .../src/cube-filter-transformer/factory.ts | 8 +++ .../not-set/not-set.spec.ts | 50 ++++++++++++++++++ .../not-set/not-set.ts | 19 +++++++ .../cube-filter-transformer/set/set.spec.ts | 51 +++++++++++++++++++ .../src/cube-filter-transformer/set/set.ts | 19 +++++++ .../src/__tests__/cube-filter-params.spec.ts | 36 +++++++++++-- 6 files changed, 178 insertions(+), 5 deletions(-) create mode 100644 meerkat-core/src/cube-filter-transformer/not-set/not-set.spec.ts create mode 100644 meerkat-core/src/cube-filter-transformer/not-set/not-set.ts create mode 100644 meerkat-core/src/cube-filter-transformer/set/set.spec.ts create mode 100644 meerkat-core/src/cube-filter-transformer/set/set.ts diff --git a/meerkat-core/src/cube-filter-transformer/factory.ts b/meerkat-core/src/cube-filter-transformer/factory.ts index c62170d3..483343fb 100644 --- a/meerkat-core/src/cube-filter-transformer/factory.ts +++ b/meerkat-core/src/cube-filter-transformer/factory.ts @@ -25,7 +25,9 @@ import { lteTransform } from './lte/lte'; import { notInDataRangeTransform } from './not-In-date-range/not-In-date-range'; import { notContainsTransform } from './not-contains/not-contains'; import { notEqualsTransform } from './not-equals/not-equals'; +import { notSetTransform } from './not-set/not-set'; import { orDuckdbCondition } from './or/or'; +import { setTransform } from './set/set'; export type CubeToParseExpressionTransform = ( query: QueryOperatorsWithInfo @@ -54,6 +56,12 @@ const cubeFilterOperatorsToDuckdb = (cubeFilter: QueryOperatorsWithInfo) => { return inDataRangeTransform(cubeFilter); case 'notInDateRange': return notInDataRangeTransform(cubeFilter); + case 'notSet': { + return notSetTransform(cubeFilter); + } + case 'set': { + return setTransform(cubeFilter); + } default: throw new Error('Could not transform the filter'); } diff --git a/meerkat-core/src/cube-filter-transformer/not-set/not-set.spec.ts b/meerkat-core/src/cube-filter-transformer/not-set/not-set.spec.ts new file mode 100644 index 00000000..f14973f1 --- /dev/null +++ b/meerkat-core/src/cube-filter-transformer/not-set/not-set.spec.ts @@ -0,0 +1,50 @@ +import { notSetTransform } from "./not-set"; + +describe("notSetTransform", () => { + it("should return the correct expression for a given query", () => { + const query = { + member: "table.column" + }; + + const expectedExpression = { + class: "OPERATOR", + type: "OPERATOR_IS_NULL", + alias: "", + children: [ + { + class: "COLUMN_REF", + type: "COLUMN_REF", + alias: "", + column_names: ["table", "column"] + } + ] + }; + + const result = notSetTransform(query); + + expect(result).toEqual(expectedExpression); + }); + it("should return the correct expression for a __ delimited query", () => { + const query = { + member: "table__column" + }; + + const expectedExpression = { + class: "OPERATOR", + type: "OPERATOR_IS_NULL", + alias: "", + children: [ + { + class: "COLUMN_REF", + type: "COLUMN_REF", + alias: "", + column_names: ["table__column"] + } + ] + }; + + const result = notSetTransform(query); + + expect(result).toEqual(expectedExpression); + }); +}); diff --git a/meerkat-core/src/cube-filter-transformer/not-set/not-set.ts b/meerkat-core/src/cube-filter-transformer/not-set/not-set.ts new file mode 100644 index 00000000..108cf2a5 --- /dev/null +++ b/meerkat-core/src/cube-filter-transformer/not-set/not-set.ts @@ -0,0 +1,19 @@ +import { CubeToParseExpressionTransform } from "../factory"; + +// @ts-ignore +export const notSetTransform: CubeToParseExpressionTransform = (query) => { + const { member } = query; + return { + class: "OPERATOR", + type: "OPERATOR_IS_NULL", + alias: "", + children: [ + { + class: "COLUMN_REF", + type: "COLUMN_REF", + alias: "", + column_names: member.split('.') + } + ] + } +} \ No newline at end of file diff --git a/meerkat-core/src/cube-filter-transformer/set/set.spec.ts b/meerkat-core/src/cube-filter-transformer/set/set.spec.ts new file mode 100644 index 00000000..af969892 --- /dev/null +++ b/meerkat-core/src/cube-filter-transformer/set/set.spec.ts @@ -0,0 +1,51 @@ +import { setTransform } from './set'; + +describe('setTransform', () => { + it('should return the correct expression object', () => { + const query = { + member: 'table.column' + }; + + const expected = { + class: 'OPERATOR', + type: 'OPERATOR_IS_NOT_NULL', + alias: '', + children: [ + { + class: 'COLUMN_REF', + type: 'COLUMN_REF', + alias: '', + column_names: ['table', 'column'] + } + ] + }; + + const result = setTransform(query); + + expect(result).toEqual(expected); + }); + + it('should handle __ delimited query', () => { + const query = { + member: 'table__column' + }; + + const expected = { + class: 'OPERATOR', + type: 'OPERATOR_IS_NOT_NULL', + alias: '', + children: [ + { + class: 'COLUMN_REF', + type: 'COLUMN_REF', + alias: '', + column_names: ['table__column'] + } + ] + }; + + const result = setTransform(query); + + expect(result).toEqual(expected); + }); +}); diff --git a/meerkat-core/src/cube-filter-transformer/set/set.ts b/meerkat-core/src/cube-filter-transformer/set/set.ts new file mode 100644 index 00000000..bbaf5496 --- /dev/null +++ b/meerkat-core/src/cube-filter-transformer/set/set.ts @@ -0,0 +1,19 @@ +import { CubeToParseExpressionTransform } from "../factory"; + +// @ts-ignore +export const setTransform: CubeToParseExpressionTransform = (query) => { + const { member } = query; + return { + class: "OPERATOR", + type: "OPERATOR_IS_NOT_NULL", + alias: "", + children: [ + { + class: "COLUMN_REF", + type: "COLUMN_REF", + alias: "", + column_names: member.split('.') + } + ] + } +} \ No newline at end of file diff --git a/meerkat-node/src/__tests__/cube-filter-params.spec.ts b/meerkat-node/src/__tests__/cube-filter-params.spec.ts index 7a940397..4570dd2f 100644 --- a/meerkat-node/src/__tests__/cube-filter-params.spec.ts +++ b/meerkat-node/src/__tests__/cube-filter-params.spec.ts @@ -42,7 +42,8 @@ describe('filter-param-tests', () => { (4, DATE '2022-03-01', 'cancelled', 40.00), (5, DATE '2022-01-28', 'completed', 80.75), (6, DATE '2022-02-15', 'pending', 120.00), - (7, DATE '2022-04-01', 'completed', 210.00);`); + (7, DATE '2022-02-15', 'pending', 120.00), + (8, DATE '2022-04-01', 'initiated', null);`); }); it('Should apply filter params to base SQL', async () => { @@ -60,7 +61,7 @@ describe('filter-param-tests', () => { const sql = await cubeQueryToSQL(query, [SCHEMA]); console.info('SQL: ', sql); const output: any = await duckdbExec(sql); - expect(output).toHaveLength(1); + expect(output).toHaveLength(2); expect(output[0].id).toBe(6); }); @@ -93,7 +94,7 @@ describe('filter-param-tests', () => { const sql = await cubeQueryToSQL(query, [SCHEMA]); console.info('SQL: ', sql); const output: any = await duckdbExec(sql); - expect(output).toHaveLength(2); + expect(output).toHaveLength(3); expect(output[0].id).toBe(6); }); @@ -107,7 +108,7 @@ describe('filter-param-tests', () => { const sql = await cubeQueryToSQL(query, [SCHEMA]); console.info('SQL: ', sql); const output: any = await duckdbExec(sql); - expect(output).toHaveLength(7); + expect(output).toHaveLength(8); }); it('Should apply true filter if filters are present but are not matching', async () => { @@ -139,6 +140,31 @@ describe('filter-param-tests', () => { const sql = await cubeQueryToSQL(query, [SCHEMA]); console.info('SQL: ', sql); const output: any = await duckdbExec(sql); - expect(output).toHaveLength(4); + expect(output).toHaveLength(5); + }); + it('Should apply notSet and set filters', async () => { + const query = { + measures: ['*'], + filters: [ + { + and: [ + { + member: 'orders.amount', + operator: 'notSet', + }, + { + member: 'orders.status', + operator: 'set', + } + ], + }, + ], + dimensions: [], + }; + + const sql = await cubeQueryToSQL(query, [SCHEMA]); + console.info('SQL: ', sql); + const output: any = await duckdbExec(sql); + expect(output).toHaveLength(1); }); }); From 2e3fbbee94a248ab2268cd492c48884006c1964f Mon Sep 17 00:00:00 2001 From: zaidjan Date: Sun, 28 Apr 2024 23:33:40 +0530 Subject: [PATCH 2/5] bumped package json --- meerkat-core/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meerkat-core/package.json b/meerkat-core/package.json index 8e327c49..8efc4ab7 100644 --- a/meerkat-core/package.json +++ b/meerkat-core/package.json @@ -1,6 +1,6 @@ { "name": "@devrev/meerkat-core", - "version": "0.0.73", + "version": "0.0.74", "dependencies": { "@swc/helpers": "~0.5.0" }, From 0d029732f3b1fe0172079c50849bd6ed0d944038 Mon Sep 17 00:00:00 2001 From: zaidjan Date: Sun, 28 Apr 2024 23:40:52 +0530 Subject: [PATCH 3/5] type fix --- meerkat-core/src/cube-filter-transformer/not-set/not-set.ts | 6 +++--- meerkat-core/src/cube-filter-transformer/set/set.ts | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/meerkat-core/src/cube-filter-transformer/not-set/not-set.ts b/meerkat-core/src/cube-filter-transformer/not-set/not-set.ts index 108cf2a5..29580057 100644 --- a/meerkat-core/src/cube-filter-transformer/not-set/not-set.ts +++ b/meerkat-core/src/cube-filter-transformer/not-set/not-set.ts @@ -1,11 +1,11 @@ +import { ExpressionClass, ExpressionType } from '../../types/duckdb-serialization-types/serialization/Expression'; import { CubeToParseExpressionTransform } from "../factory"; -// @ts-ignore export const notSetTransform: CubeToParseExpressionTransform = (query) => { const { member } = query; return { - class: "OPERATOR", - type: "OPERATOR_IS_NULL", + class: ExpressionClass.OPERATOR, + type: ExpressionType.OPERATOR_IS_NULL, alias: "", children: [ { diff --git a/meerkat-core/src/cube-filter-transformer/set/set.ts b/meerkat-core/src/cube-filter-transformer/set/set.ts index bbaf5496..d044b5d6 100644 --- a/meerkat-core/src/cube-filter-transformer/set/set.ts +++ b/meerkat-core/src/cube-filter-transformer/set/set.ts @@ -1,11 +1,11 @@ +import { ExpressionClass, ExpressionType } from '../../types/duckdb-serialization-types/serialization/Expression'; import { CubeToParseExpressionTransform } from "../factory"; -// @ts-ignore export const setTransform: CubeToParseExpressionTransform = (query) => { const { member } = query; return { - class: "OPERATOR", - type: "OPERATOR_IS_NOT_NULL", + class: ExpressionClass.OPERATOR, + type: ExpressionType.OPERATOR_IS_NOT_NULL, alias: "", children: [ { From e177402c99daa21c498817c8b8d1a51063b82612 Mon Sep 17 00:00:00 2001 From: zaidjan Date: Mon, 29 Apr 2024 00:33:14 +0530 Subject: [PATCH 4/5] test update --- .../src/utils/key-from-measures-dimension.ts | 1 - meerkat-node/src/__tests__/test-data.ts | 134 +++++++++++++++++- 2 files changed, 133 insertions(+), 2 deletions(-) diff --git a/meerkat-core/src/utils/key-from-measures-dimension.ts b/meerkat-core/src/utils/key-from-measures-dimension.ts index 12bb563f..8b1c8856 100644 --- a/meerkat-core/src/utils/key-from-measures-dimension.ts +++ b/meerkat-core/src/utils/key-from-measures-dimension.ts @@ -7,7 +7,6 @@ export const getMemberInfoFromTableSchema = ( let memberInfo: Measure | Dimension | undefined; const memberKeyName = memberKey.split('.')[1]; - console.info('memberKeyName', memberKeyName, memberKey); /** * Finding the table key from the measures. diff --git a/meerkat-node/src/__tests__/test-data.ts b/meerkat-node/src/__tests__/test-data.ts index 4fdc3087..e5feafd5 100644 --- a/meerkat-node/src/__tests__/test-data.ts +++ b/meerkat-node/src/__tests__/test-data.ts @@ -20,7 +20,8 @@ INSERT INTO orders VALUES (8, '5', '1', '2022-05-02', 65), (9, '5', '2', '2022-05-05', 85), (10, '6', '3', '2022-06-01', 120), -(11, '6aa6', '3', '2024-06-01', 0); +(11, '6aa6', '3', '2024-06-01', 0), +(12, NULL, '3', '2024-07-01', 100); `; export const TABLE_SCHEMA = { @@ -121,6 +122,10 @@ export const TEST_DATA = [ orders__customer_id: '6aa6', orders__total_order_amount: 0, }, + { + orders__customer_id: null, + orders__total_order_amount: 100, + } ], }, { @@ -395,6 +400,15 @@ export const TEST_DATA = [ order_amount: 120.0, orders__order_amount: 120.0, }, + { + "customer_id": null, + "order_amount": 100, + "order_date": "2024-07-01T00:00:00.000Z", + "order_id": 12, + "orders__order_amount": 100, + "orders__order_date": undefined, + "product_id": "3", + } ], }, { @@ -558,6 +572,14 @@ export const TEST_DATA = [ orders__order_date: '2024-06-01', order_amount: 0.0, }, + { + "customer_id": null, + "order_amount": 100, + "order_date": "2024-07-01T00:00:00.000Z", + "order_id": 12, + "orders__order_date": "2024-07-01T00:00:00.000Z", + "product_id": "3", + } ], }, // { @@ -708,4 +730,114 @@ export const TEST_DATA = [ }, ], }, + { + testName: 'Set', + expectedSQL: `SELECT orders.* FROM (SELECT *, orders.order_amount AS orders__order_amount, product_id AS orders__product_id FROM (select * from orders) AS orders) AS orders WHERE ((orders__order_amount IS NOT NULL) AND (orders__product_id = '3'))`, + cubeInput: { + measures: ['*'], + filters: [ + { + and: [ + { + member: 'orders.order_amount', + operator: 'set', + }, + { + member: 'orders.product_id', + operator: 'equals', + values: ['3'] + } + ], + }, + ], + dimensions: [], + }, + expectedOutput: [ + { + "customer_id": "2", + "order_amount": 25, + "order_date": "2022-02-01T00:00:00.000Z", + "order_id": 3, + "orders__order_amount": 25, + "orders__order_date": undefined, + "orders__product_id": "3", + "product_id": "3", + }, + { + "customer_id": "4", + "order_amount": 90, + "order_date": "2022-05-01T00:00:00.000Z", + "order_id": 7, + "orders__order_amount": 90, + "orders__order_date": undefined, + "orders__product_id": "3", + "product_id": "3", + }, + { + "customer_id": "6", + "order_amount": 120, + "order_date": "2022-06-01T00:00:00.000Z", + "order_id": 10, + "orders__order_amount": 120, + "orders__order_date": undefined, + "orders__product_id": "3", + "product_id": "3", + }, + { + "customer_id": "6aa6", + "order_amount": 0, + "order_date": "2024-06-01T00:00:00.000Z", + "order_id": 11, + "orders__order_amount": 0, + "orders__order_date": undefined, + "orders__product_id": "3", + "product_id": "3", + }, + { + "customer_id": null, + "order_amount": 100, + "order_date": "2024-07-01T00:00:00.000Z", + "order_id": 12, + "orders__order_amount": 100, + "orders__order_date": undefined, + "orders__product_id": "3", + "product_id": "3", + }, + ], + }, + { + testName: 'Not Set', + expectedSQL: `SELECT orders.* FROM (SELECT *, customer_id AS orders__customer_id, product_id AS orders__product_id FROM (select * from orders) AS orders) AS orders WHERE ((orders__customer_id IS NULL) AND (orders__product_id = '3'))`, + cubeInput: { + measures: ['*'], + filters: [ + { + and: [ + { + member: 'orders.customer_id', + operator: 'notSet', + }, + { + member: 'orders.product_id', + operator: 'equals', + values: ['3'] + } + ], + }, + ], + dimensions: [], + }, + expectedOutput: [ + { + "orders__customer_id": null, + "customer_id": null, + "order_amount": 100, + "order_date": "2024-07-01T00:00:00.000Z", + "order_id": 12, + "orders__order_date": undefined, + "orders__product_id": "3", + "product_id": "3", + } + ], + }, ]; From 498a310962fa6cb4bed09ac01d1eed94476acfbc Mon Sep 17 00:00:00 2001 From: zaidjan Date: Mon, 29 Apr 2024 00:34:01 +0530 Subject: [PATCH 5/5] bumped node and browser --- meerkat-browser/package.json | 2 +- meerkat-node/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/meerkat-browser/package.json b/meerkat-browser/package.json index c9781ce3..be6ba597 100644 --- a/meerkat-browser/package.json +++ b/meerkat-browser/package.json @@ -1,6 +1,6 @@ { "name": "@devrev/meerkat-browser", - "version": "0.0.71", + "version": "0.0.72", "dependencies": { "@swc/helpers": "~0.5.0", "@devrev/meerkat-core": "*", diff --git a/meerkat-node/package.json b/meerkat-node/package.json index 7bd15137..2a01d26f 100644 --- a/meerkat-node/package.json +++ b/meerkat-node/package.json @@ -1,6 +1,6 @@ { "name": "@devrev/meerkat-node", - "version": "0.0.71", + "version": "0.0.72", "dependencies": { "@swc/helpers": "~0.5.0", "@devrev/meerkat-core": "*",