diff --git a/meerkat-browser/package.json b/meerkat-browser/package.json index c9781ce3..be6ba597 100644 --- a/meerkat-browser/package.json +++ b/meerkat-browser/package.json @@ -1,6 +1,6 @@ { "name": "@devrev/meerkat-browser", - "version": "0.0.71", + "version": "0.0.72", "dependencies": { "@swc/helpers": "~0.5.0", "@devrev/meerkat-core": "*", diff --git a/meerkat-core/package.json b/meerkat-core/package.json index 8e327c49..8efc4ab7 100644 --- a/meerkat-core/package.json +++ b/meerkat-core/package.json @@ -1,6 +1,6 @@ { "name": "@devrev/meerkat-core", - "version": "0.0.73", + "version": "0.0.74", "dependencies": { "@swc/helpers": "~0.5.0" }, diff --git a/meerkat-core/src/cube-filter-transformer/factory.ts b/meerkat-core/src/cube-filter-transformer/factory.ts index c62170d3..483343fb 100644 --- a/meerkat-core/src/cube-filter-transformer/factory.ts +++ b/meerkat-core/src/cube-filter-transformer/factory.ts @@ -25,7 +25,9 @@ import { lteTransform } from './lte/lte'; import { notInDataRangeTransform } from './not-In-date-range/not-In-date-range'; import { notContainsTransform } from './not-contains/not-contains'; import { notEqualsTransform } from './not-equals/not-equals'; +import { notSetTransform } from './not-set/not-set'; import { orDuckdbCondition } from './or/or'; +import { setTransform } from './set/set'; export type CubeToParseExpressionTransform = ( query: QueryOperatorsWithInfo @@ -54,6 +56,12 @@ const cubeFilterOperatorsToDuckdb = (cubeFilter: QueryOperatorsWithInfo) => { return inDataRangeTransform(cubeFilter); case 'notInDateRange': return notInDataRangeTransform(cubeFilter); + case 'notSet': { + return notSetTransform(cubeFilter); + } + case 'set': { + return setTransform(cubeFilter); + } default: throw new Error('Could not transform the filter'); } diff --git a/meerkat-core/src/cube-filter-transformer/not-set/not-set.spec.ts b/meerkat-core/src/cube-filter-transformer/not-set/not-set.spec.ts new file mode 100644 index 00000000..f14973f1 --- /dev/null +++ b/meerkat-core/src/cube-filter-transformer/not-set/not-set.spec.ts @@ -0,0 +1,50 @@ +import { notSetTransform } from "./not-set"; + +describe("notSetTransform", () => { + it("should return the correct expression for a given query", () => { + const query = { + member: "table.column" + }; + + const expectedExpression = { + class: "OPERATOR", + type: "OPERATOR_IS_NULL", + alias: "", + children: [ + { + class: "COLUMN_REF", + type: "COLUMN_REF", + alias: "", + column_names: ["table", "column"] + } + ] + }; + + const result = notSetTransform(query); + + expect(result).toEqual(expectedExpression); + }); + it("should return the correct expression for a __ delimited query", () => { + const query = { + member: "table__column" + }; + + const expectedExpression = { + class: "OPERATOR", + type: "OPERATOR_IS_NULL", + alias: "", + children: [ + { + class: "COLUMN_REF", + type: "COLUMN_REF", + alias: "", + column_names: ["table__column"] + } + ] + }; + + const result = notSetTransform(query); + + expect(result).toEqual(expectedExpression); + }); +}); diff --git a/meerkat-core/src/cube-filter-transformer/not-set/not-set.ts b/meerkat-core/src/cube-filter-transformer/not-set/not-set.ts new file mode 100644 index 00000000..29580057 --- /dev/null +++ b/meerkat-core/src/cube-filter-transformer/not-set/not-set.ts @@ -0,0 +1,19 @@ +import { ExpressionClass, ExpressionType } from '../../types/duckdb-serialization-types/serialization/Expression'; +import { CubeToParseExpressionTransform } from "../factory"; + +export const notSetTransform: CubeToParseExpressionTransform = (query) => { + const { member } = query; + return { + class: ExpressionClass.OPERATOR, + type: ExpressionType.OPERATOR_IS_NULL, + alias: "", + children: [ + { + class: "COLUMN_REF", + type: "COLUMN_REF", + alias: "", + column_names: member.split('.') + } + ] + } +} \ No newline at end of file diff --git a/meerkat-core/src/cube-filter-transformer/set/set.spec.ts b/meerkat-core/src/cube-filter-transformer/set/set.spec.ts new file mode 100644 index 00000000..af969892 --- /dev/null +++ b/meerkat-core/src/cube-filter-transformer/set/set.spec.ts @@ -0,0 +1,51 @@ +import { setTransform } from './set'; + +describe('setTransform', () => { + it('should return the correct expression object', () => { + const query = { + member: 'table.column' + }; + + const expected = { + class: 'OPERATOR', + type: 'OPERATOR_IS_NOT_NULL', + alias: '', + children: [ + { + class: 'COLUMN_REF', + type: 'COLUMN_REF', + alias: '', + column_names: ['table', 'column'] + } + ] + }; + + const result = setTransform(query); + + expect(result).toEqual(expected); + }); + + it('should handle __ delimited query', () => { + const query = { + member: 'table__column' + }; + + const expected = { + class: 'OPERATOR', + type: 'OPERATOR_IS_NOT_NULL', + alias: '', + children: [ + { + class: 'COLUMN_REF', + type: 'COLUMN_REF', + alias: '', + column_names: ['table__column'] + } + ] + }; + + const result = setTransform(query); + + expect(result).toEqual(expected); + }); +}); diff --git a/meerkat-core/src/cube-filter-transformer/set/set.ts b/meerkat-core/src/cube-filter-transformer/set/set.ts new file mode 100644 index 00000000..d044b5d6 --- /dev/null +++ b/meerkat-core/src/cube-filter-transformer/set/set.ts @@ -0,0 +1,19 @@ +import { ExpressionClass, ExpressionType } from '../../types/duckdb-serialization-types/serialization/Expression'; +import { CubeToParseExpressionTransform } from "../factory"; + +export const setTransform: CubeToParseExpressionTransform = (query) => { + const { member } = query; + return { + class: ExpressionClass.OPERATOR, + type: ExpressionType.OPERATOR_IS_NOT_NULL, + alias: "", + children: [ + { + class: "COLUMN_REF", + type: "COLUMN_REF", + alias: "", + column_names: member.split('.') + } + ] + } +} \ No newline at end of file diff --git a/meerkat-core/src/utils/key-from-measures-dimension.ts b/meerkat-core/src/utils/key-from-measures-dimension.ts index 12bb563f..8b1c8856 100644 --- a/meerkat-core/src/utils/key-from-measures-dimension.ts +++ b/meerkat-core/src/utils/key-from-measures-dimension.ts @@ -7,7 +7,6 @@ export const getMemberInfoFromTableSchema = ( let memberInfo: Measure | Dimension | undefined; const memberKeyName = memberKey.split('.')[1]; - console.info('memberKeyName', memberKeyName, memberKey); /** * Finding the table key from the measures. diff --git a/meerkat-node/package.json b/meerkat-node/package.json index 7bd15137..2a01d26f 100644 --- a/meerkat-node/package.json +++ b/meerkat-node/package.json @@ -1,6 +1,6 @@ { "name": "@devrev/meerkat-node", - "version": "0.0.71", + "version": "0.0.72", "dependencies": { "@swc/helpers": "~0.5.0", "@devrev/meerkat-core": "*", diff --git a/meerkat-node/src/__tests__/cube-filter-params.spec.ts b/meerkat-node/src/__tests__/cube-filter-params.spec.ts index 7a940397..4570dd2f 100644 --- a/meerkat-node/src/__tests__/cube-filter-params.spec.ts +++ b/meerkat-node/src/__tests__/cube-filter-params.spec.ts @@ -42,7 +42,8 @@ describe('filter-param-tests', () => { (4, DATE '2022-03-01', 'cancelled', 40.00), (5, DATE '2022-01-28', 'completed', 80.75), (6, DATE '2022-02-15', 'pending', 120.00), - (7, DATE '2022-04-01', 'completed', 210.00);`); + (7, DATE '2022-02-15', 'pending', 120.00), + (8, DATE '2022-04-01', 'initiated', null);`); }); it('Should apply filter params to base SQL', async () => { @@ -60,7 +61,7 @@ describe('filter-param-tests', () => { const sql = await cubeQueryToSQL(query, [SCHEMA]); console.info('SQL: ', sql); const output: any = await duckdbExec(sql); - expect(output).toHaveLength(1); + expect(output).toHaveLength(2); expect(output[0].id).toBe(6); }); @@ -93,7 +94,7 @@ describe('filter-param-tests', () => { const sql = await cubeQueryToSQL(query, [SCHEMA]); console.info('SQL: ', sql); const output: any = await duckdbExec(sql); - expect(output).toHaveLength(2); + expect(output).toHaveLength(3); expect(output[0].id).toBe(6); }); @@ -107,7 +108,7 @@ describe('filter-param-tests', () => { const sql = await cubeQueryToSQL(query, [SCHEMA]); console.info('SQL: ', sql); const output: any = await duckdbExec(sql); - expect(output).toHaveLength(7); + expect(output).toHaveLength(8); }); it('Should apply true filter if filters are present but are not matching', async () => { @@ -139,6 +140,31 @@ describe('filter-param-tests', () => { const sql = await cubeQueryToSQL(query, [SCHEMA]); console.info('SQL: ', sql); const output: any = await duckdbExec(sql); - expect(output).toHaveLength(4); + expect(output).toHaveLength(5); + }); + it('Should apply notSet and set filters', async () => { + const query = { + measures: ['*'], + filters: [ + { + and: [ + { + member: 'orders.amount', + operator: 'notSet', + }, + { + member: 'orders.status', + operator: 'set', + } + ], + }, + ], + dimensions: [], + }; + + const sql = await cubeQueryToSQL(query, [SCHEMA]); + console.info('SQL: ', sql); + const output: any = await duckdbExec(sql); + expect(output).toHaveLength(1); }); }); diff --git a/meerkat-node/src/__tests__/test-data.ts b/meerkat-node/src/__tests__/test-data.ts index 4fdc3087..e5feafd5 100644 --- a/meerkat-node/src/__tests__/test-data.ts +++ b/meerkat-node/src/__tests__/test-data.ts @@ -20,7 +20,8 @@ INSERT INTO orders VALUES (8, '5', '1', '2022-05-02', 65), (9, '5', '2', '2022-05-05', 85), (10, '6', '3', '2022-06-01', 120), -(11, '6aa6', '3', '2024-06-01', 0); +(11, '6aa6', '3', '2024-06-01', 0), +(12, NULL, '3', '2024-07-01', 100); `; export const TABLE_SCHEMA = { @@ -121,6 +122,10 @@ export const TEST_DATA = [ orders__customer_id: '6aa6', orders__total_order_amount: 0, }, + { + orders__customer_id: null, + orders__total_order_amount: 100, + } ], }, { @@ -395,6 +400,15 @@ export const TEST_DATA = [ order_amount: 120.0, orders__order_amount: 120.0, }, + { + "customer_id": null, + "order_amount": 100, + "order_date": "2024-07-01T00:00:00.000Z", + "order_id": 12, + "orders__order_amount": 100, + "orders__order_date": undefined, + "product_id": "3", + } ], }, { @@ -558,6 +572,14 @@ export const TEST_DATA = [ orders__order_date: '2024-06-01', order_amount: 0.0, }, + { + "customer_id": null, + "order_amount": 100, + "order_date": "2024-07-01T00:00:00.000Z", + "order_id": 12, + "orders__order_date": "2024-07-01T00:00:00.000Z", + "product_id": "3", + } ], }, // { @@ -708,4 +730,114 @@ export const TEST_DATA = [ }, ], }, + { + testName: 'Set', + expectedSQL: `SELECT orders.* FROM (SELECT *, orders.order_amount AS orders__order_amount, product_id AS orders__product_id FROM (select * from orders) AS orders) AS orders WHERE ((orders__order_amount IS NOT NULL) AND (orders__product_id = '3'))`, + cubeInput: { + measures: ['*'], + filters: [ + { + and: [ + { + member: 'orders.order_amount', + operator: 'set', + }, + { + member: 'orders.product_id', + operator: 'equals', + values: ['3'] + } + ], + }, + ], + dimensions: [], + }, + expectedOutput: [ + { + "customer_id": "2", + "order_amount": 25, + "order_date": "2022-02-01T00:00:00.000Z", + "order_id": 3, + "orders__order_amount": 25, + "orders__order_date": undefined, + "orders__product_id": "3", + "product_id": "3", + }, + { + "customer_id": "4", + "order_amount": 90, + "order_date": "2022-05-01T00:00:00.000Z", + "order_id": 7, + "orders__order_amount": 90, + "orders__order_date": undefined, + "orders__product_id": "3", + "product_id": "3", + }, + { + "customer_id": "6", + "order_amount": 120, + "order_date": "2022-06-01T00:00:00.000Z", + "order_id": 10, + "orders__order_amount": 120, + "orders__order_date": undefined, + "orders__product_id": "3", + "product_id": "3", + }, + { + "customer_id": "6aa6", + "order_amount": 0, + "order_date": "2024-06-01T00:00:00.000Z", + "order_id": 11, + "orders__order_amount": 0, + "orders__order_date": undefined, + "orders__product_id": "3", + "product_id": "3", + }, + { + "customer_id": null, + "order_amount": 100, + "order_date": "2024-07-01T00:00:00.000Z", + "order_id": 12, + "orders__order_amount": 100, + "orders__order_date": undefined, + "orders__product_id": "3", + "product_id": "3", + }, + ], + }, + { + testName: 'Not Set', + expectedSQL: `SELECT orders.* FROM (SELECT *, customer_id AS orders__customer_id, product_id AS orders__product_id FROM (select * from orders) AS orders) AS orders WHERE ((orders__customer_id IS NULL) AND (orders__product_id = '3'))`, + cubeInput: { + measures: ['*'], + filters: [ + { + and: [ + { + member: 'orders.customer_id', + operator: 'notSet', + }, + { + member: 'orders.product_id', + operator: 'equals', + values: ['3'] + } + ], + }, + ], + dimensions: [], + }, + expectedOutput: [ + { + "orders__customer_id": null, + "customer_id": null, + "order_amount": 100, + "order_date": "2024-07-01T00:00:00.000Z", + "order_id": 12, + "orders__order_date": undefined, + "orders__product_id": "3", + "product_id": "3", + } + ], + }, ];