Skip to content

UBERF-9560: Filter query fixes #8161

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Mar 7, 2025
Merged

UBERF-9560: Filter query fixes #8161

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 45 additions & 1 deletion server/postgres/src/__tests__/conversion.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { convertArrayParams, decodeArray } from '../utils'
import { convertArrayParams, decodeArray, filterProjection } from '../utils'

describe('array conversion', () => {
it('should handle undefined parameters', () => {
Expand Down Expand Up @@ -55,3 +55,47 @@ describe('array decoding', () => {
expect(decodeArray('{"first \\"quote\\"","second \\"quote\\""}')).toEqual(['first "quote"', 'second "quote"'])
})
})

describe('projection', () => {
it('mixin query projection', () => {
const data = {
'638611f18894c91979399ef3': {
Источник_6386125d8894c91979399eff: 'Workable'
},
attachments: 1,
avatar: null,
avatarProps: null,
avatarType: 'color',
channels: 3,
city: 'Poland',
docUpdateMessages: 31,
name: 'Mulkuha,Muklyi',
'notification:mixin:Collaborators': {
collaborators: []
},
'recruit:mixin:Candidate': {
Title_63f38419efefd99805238bbd: 'Backend-RoR',
Trash_64493626f9b50e77bf82d231: 'Нет',
__mixin: 'true',
applications: 1,
onsite: null,
remote: null,
skills: 18,
title: '',
Опытработы_63860d5c8894c91979399e73: '2018',
Уровеньанглийского_63860d038894c91979399e6f: 'UPPER'
}
}
const projected = filterProjection<any>(data, {
'recruit:mixin:Candidate.Уровеньанглийского_63860d038894c91979399e6f': 1,
_class: 1,
space: 1,
modifiedOn: 1
})
expect(projected).toEqual({
'recruit:mixin:Candidate': {
Уровеньанглийского_63860d038894c91979399e6f: 'UPPER'
}
})
})
})
83 changes: 59 additions & 24 deletions server/postgres/src/storage.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,12 +60,12 @@ import core, {
type WorkspaceId
} from '@hcengineering/core'
import {
calcHashHash,
type DbAdapter,
type DbAdapterHandler,
type DomainHelperOperations,
type ServerFindOptions,
type TxAdapter,
calcHashHash
type TxAdapter
} from '@hcengineering/server-core'
import type postgres from 'postgres'
import { createDBClient, createGreenDBClient, type DBClient } from './client'
Expand Down Expand Up @@ -1304,7 +1304,11 @@ abstract class PostgresAdapterBase implements DbAdapter {
private translateQueryValue (vars: ValuesVariables, tkey: string, value: any, type: ValueType): string | undefined {
const tkeyData = tkey.includes('data') && (tkey.includes('->') || tkey.includes('#>>'))
if (tkeyData && (Array.isArray(value) || (typeof value !== 'object' && typeof value !== 'string'))) {
value = Array.isArray(value) ? value.map((it) => (it == null ? null : `${it}`)) : `${value}`
value = Array.isArray(value)
? value.map((it) => (it == null ? null : `${it}`))
: value == null
? null
: `${value}`
}

if (value === null) {
Expand All @@ -1315,76 +1319,87 @@ abstract class PostgresAdapterBase implements DbAdapter {
for (const operator in value) {
let val = value[operator]
if (tkeyData && (Array.isArray(val) || (typeof val !== 'object' && typeof val !== 'string'))) {
val = Array.isArray(val) ? val.map((it) => (it == null ? null : `${it}`)) : `${val}`
val = Array.isArray(val) ? val.map((it) => (it == null ? null : `${it}`)) : val == null ? null : `${val}`
}

let valType = inferType(val)
const { tlkey, arrowCount } = prepareJsonValue(tkey, valType)
if (arrowCount > 0 && valType === '::text') {
valType = ''
}

switch (operator) {
case '$ne':
if (val === null) {
res.push(`${tkey} IS NOT NULL`)
if (val == null) {
res.push(`${tlkey} IS NOT NULL`)
} else {
res.push(`${tkey} != ${vars.add(val, inferType(val))}`)
res.push(`${tlkey} != ${vars.add(val, valType)}`)
}
break
case '$gt':
res.push(`${tkey} > ${vars.add(val, inferType(val))}`)
res.push(`${tlkey} > ${vars.add(val, valType)}`)
break
case '$gte':
res.push(`${tkey} >= ${vars.add(val, inferType(val))}`)
res.push(`${tlkey} >= ${vars.add(val, valType)}`)
break
case '$lt':
res.push(`${tkey} < ${vars.add(val, inferType(val))}`)
res.push(`${tlkey} < ${vars.add(val, valType)}`)
break
case '$lte':
res.push(`${tkey} <= ${vars.add(val, inferType(val))}`)
res.push(`${tlkey} <= ${vars.add(val, valType)}`)
break
case '$in':
switch (type) {
case 'common':
if (Array.isArray(val) && val.includes(null)) {
const vv = vars.addArray(val, inferType(val))
res.push(`(${tkey} = ANY(${vv}) OR ${tkey} IS NULL)`)
const vv = vars.addArray(val, valType)
res.push(`(${tlkey} = ANY(${vv}) OR ${tkey} IS NULL)`)
} else {
if (val.length > 0) {
res.push(`${tkey} = ANY(${vars.addArray(val, inferType(val))})`)
res.push(`${tlkey} = ANY(${vars.addArray(val, valType)})`)
} else {
res.push(`${tkey} IN ('NULL')`)
res.push(`${tlkey} IN ('NULL')`)
}
}
break
case 'array':
{
const vv = vars.addArrayI(val, inferType(val))
const vv = vars.addArrayI(val, valType)
res.push(`${tkey} && ${vv}`)
}
break
case 'dataArray':
{
const vv = vars.addArrayI(val, inferType(val))
const vv = vars.addArrayI(val, valType)
res.push(`${tkey} ?| ${vv}`)
}
break
}
break
case '$nin':
if (Array.isArray(val) && val.includes(null)) {
res.push(`(${tkey} != ALL(${vars.addArray(val, inferType(val))}) AND ${tkey} IS NOT NULL)`)
res.push(`(${tlkey} != ALL(${vars.addArray(val, valType)}) AND ${tkey} IS NOT NULL)`)
} else if (Array.isArray(val) && val.length > 0) {
res.push(`${tkey} != ALL(${vars.addArray(val, inferType(val))})`)
res.push(`${tlkey} != ALL(${vars.addArray(val, valType)})`)
}
break
case '$like':
res.push(`${tkey} ILIKE ${vars.add(val, inferType(val))}`)
res.push(`${tlkey} ILIKE ${vars.add(val, valType)}`)
break
case '$exists':
res.push(`${tkey} IS ${val === true || val === 'true' ? 'NOT NULL' : 'NULL'}`)
res.push(`${tlkey} IS ${val === true || val === 'true' ? 'NOT NULL' : 'NULL'}`)
break
case '$regex':
res.push(`${tkey} SIMILAR TO ${vars.add(val, inferType(val))}`)
res.push(`${tlkey} SIMILAR TO ${vars.add(val, valType)}`)
break
case '$options':
break
case '$all':
res.push(`${tkey} @> ${vars.addArray(value, inferType(value))}`)
if (arrowCount > 0) {
res.push(`${tkey} @> '${JSON.stringify(val)}'::jsonb`)
} else {
res.push(`${tkey} @> ${vars.addArray(val, valType)}`)
}
break
default:
res.push(`${tkey} @> '[${JSON.stringify(value)}]'`)
Expand All @@ -1394,8 +1409,13 @@ abstract class PostgresAdapterBase implements DbAdapter {
return res.length === 0 ? undefined : res.join(' AND ')
}

let valType = inferType(value)
const { tlkey, arrowCount } = prepareJsonValue(tkey, valType)
if (arrowCount > 0 && valType === '::text') {
valType = ''
}
return type === 'common'
? `${tkey} = ${vars.add(value, inferType(value))}`
? `${tlkey} = ${vars.add(value, valType)}`
: type === 'array'
? `${tkey} @> '${typeof value === 'string' ? '{"' + value + '"}' : value}'`
: `${tkey} @> '${typeof value === 'string' ? '"' + value + '"' : value}'`
Expand Down Expand Up @@ -2093,6 +2113,21 @@ class PostgresTxAdapter extends PostgresAdapterBase implements TxAdapter {
return this.stripHash(systemTx.concat(userTx)) as Tx[]
}
}
function prepareJsonValue (tkey: string, valType: string): { tlkey: string, arrowCount: number } {
if (valType === '::string') {
valType = '' // No need to add a string conversion
}
const arrowCount = (tkey.match(/->/g) ?? []).length
// We need to convert to type without array if pressent
let tlkey = arrowCount > 0 ? `(${tkey})${valType.replace('[]', '')}` : tkey

if (arrowCount > 0) {
// We need to replace only the last -> to ->>
tlkey = arrowCount === 1 ? tlkey.replace('->', '->>') : tlkey.replace(/->(?!.*->)/, '->>')
}
return { tlkey, arrowCount }
}

/**
* @public
*/
Expand Down
34 changes: 27 additions & 7 deletions server/postgres/src/utils.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -556,6 +556,30 @@ export function convertArrayParams (parameters?: ParameterOrJSON<any>[]): any[]
})
}

export function filterProjection<T extends Doc> (data: any, projection: Projection<T> | undefined): any {
for (const key in data) {
if (!Object.prototype.hasOwnProperty.call(projection, key) || (projection as any)[key] === 0) {
// check nested projections in case of object
let value = data[key]
if (typeof value === 'object' && !Array.isArray(value) && value != null) {
// We need to filter projection for nested objects
const innerP = Object.entries(projection as any)
.filter((it) => it[0].startsWith(key))
.map((it) => [it[0].substring(key.length + 1), it[1]])
if (innerP.length > 0) {
value = filterProjection(value, Object.fromEntries(innerP))
data[key] = value
continue
}
}

// eslint-disable-next-line @typescript-eslint/no-dynamic-delete
delete data[key]
}
}
return data
}

export function parseDocWithProjection<T extends Doc> (
doc: DBDoc,
domain: string,
Expand All @@ -577,16 +601,12 @@ export function parseDocWithProjection<T extends Doc> (
;(rest as any)[key] = decodeArray((rest as any)[key])
}
}
let resultData = data
if (projection !== undefined) {
for (const key in data) {
if (!Object.prototype.hasOwnProperty.call(projection, key) || (projection as any)[key] === 0) {
// eslint-disable-next-line @typescript-eslint/no-dynamic-delete
delete data[key]
}
}
resultData = filterProjection(data, projection)
}
const res = {
...data,
...resultData,
...rest
} as any as T

Expand Down