Merge remote-tracking branch 'origin/main' into reduce-memory-usage

Author: rkistner

.changeset/wet-gorillas-remember.md

@@ -0,0 +1,5 @@
+---
+'@powersync/service-sync-rules': minor
+---
+
+Expand supported combinations of the IN operator

.env.template

@@ -1,3 +1,3 @@
 # Connections for tests
-MONGO_TEST_UR="mongodb://localhost:27017/powersync_test"
+MONGO_TEST_URL="mongodb://localhost:27017/powersync_test"
 PG_TEST_URL="postgres://postgres:postgres@localhost:5432/powersync_test"

DEVELOP.md

@@ -20,7 +20,7 @@ pnpm build
 
 The PowerSync service requires Postgres and MongoDB server connections. These configuration details can be specified in a `powersync.yaml` (or JSON) configuration file.
 
-See the [Self hosting demo](https://github.com/powersync-ja/self-host-demo) for demos of starting these services.
+See the [self-hosting demo](https://github.com/powersync-ja/self-host-demo) for demos of starting these services.
 
 A quick method for running all required services with a handy backend and frontend is to run the following in a checked-out `self-host-demo` folder.
 
@@ -65,7 +65,7 @@ Some tests for these packages require a connection to MongoDB and Postgres. Conn
 These can be set in a terminal/shell
 
 ```bash
-export MONGO_TEST_UR="mongodb://localhost:27017/powersync_test"
+export MONGO_TEST_URL="mongodb://localhost:27017/powersync_test"
 export PG_TEST_URL="postgres://postgres:postgres@localhost:5432/powersync_test"
 ```
 

README.md

@@ -2,11 +2,11 @@
   <a href="https://www.powersync.com" target="_blank"><img src="https://github.com/powersync-ja/.github/assets/7372448/d2538c43-c1a0-4c47-9a76-41462dba484f"/></a>
 </p>
 
-_[PowerSync](https://www.powersync.com) is a Postgres-SQLite sync engine, which helps developers to create local-first real-time reactive apps that work seamlessly both online and offline._
+*[PowerSync](https://www.powersync.com) is a sync engine for building local-first apps with instantly-responsive UI/UX and simplified state transfer. Syncs between SQLite on the client-side and Postgres or MongoDB on the server-side (MySQL coming soon).*
 
 # PowerSync Service
 
-`powersync-service` is the monorepo for the core [PowerSync service](https://docs.powersync.com/architecture/powersync-service).
+`powersync-service` is the monorepo for the core [PowerSync Service](https://docs.powersync.com/architecture/powersync-service).
 
 The service can be started using the public Docker image. See the image [notes](./service/README.md)
 
@@ -19,7 +19,7 @@ The service can be started using the public Docker image. See the image [notes](
 
 - [packages/jpgwire](./packages/jpgwire/README.md)
 
-  - Customized version of [pgwire](https://www.npmjs.com/package/pgwire?activeTab=dependencies)
+  - Customized version of [pgwire](https://www.npmjs.com/package/pgwire?activeTab=dependencies) (used with Postgres)
 
 - [packages/jsonbig](./packages/jsonbig/README.md)
 
@@ -33,8 +33,8 @@ The service can be started using the public Docker image. See the image [notes](
 
   - Library containing logic for PowerSync sync rules
 
-- [packages/types](./packages/types/README.md)
-  - Type definitions for the PowerSync service
+- [packages/types](./packages/types/)
+  - Type definitions for the PowerSync Service
 
 ## Libraries
 
@@ -46,7 +46,7 @@ The service can be started using the public Docker image. See the image [notes](
 
 - [service](./service/README.md)
 
-Contains the PowerSync service code. This project is used to build the `journeyapps/powersync-service` Docker image.
+Contains the PowerSync Service code. This project is used to build the `journeyapps/powersync-service` Docker image.
 
 ## Docs
 

packages/sync-rules/src/StaticSqlParameterQuery.ts

@@ -117,7 +117,7 @@ export class StaticSqlParameterQuery {
 
   get usesUnauthenticatedRequestParameters(): boolean {
     // select where request.parameters() ->> 'include_comments'
-    const unauthenticatedFilter = this.filter!.usesUnauthenticatedRequestParameters;
+    const unauthenticatedFilter = this.filter?.usesUnauthenticatedRequestParameters;
 
     // select request.parameters() ->> 'project_id'
     const unauthenticatedExtractor =

packages/sync-rules/src/sql_filters.ts

@@ -4,6 +4,7 @@ import { ExpressionType, TYPE_NONE } from './ExpressionType.js';
 import { SqlRuleError } from './errors.js';
 import {
   BASIC_OPERATORS,
+  OPERATOR_IN,
   OPERATOR_IS_NOT_NULL,
   OPERATOR_IS_NULL,
   OPERATOR_JSON_EXTRACT_JSON,
@@ -302,13 +303,18 @@ export class SqlTools {
           throw new Error('Unexpected');
         }
       } else if (op == 'IN') {
-        // Options:
-        //  static IN static
-        //  parameterValue IN static
-
-        if (isRowValueClause(leftFilter) && isRowValueClause(rightFilter)) {
-          // static1 IN static2
-          return compileStaticOperator(op, leftFilter, rightFilter);
+        // Special cases:
+        //  parameterValue IN rowValue
+        //  rowValue IN parameterValue
+        // All others are handled by standard function composition
+
+        const composeType = this.getComposeType(OPERATOR_IN, [leftFilter, rightFilter], [left, right]);
+        if (composeType.errorClause != null) {
+          return composeType.errorClause;
+        } else if (composeType.argsType != null) {
+          // This is a standard supported configuration, takes precedence over
+          // the special cases below.
+          return this.composeFunction(OPERATOR_IN, [leftFilter, rightFilter], [left, right]);
         } else if (isParameterValueClause(leftFilter) && isRowValueClause(rightFilter)) {
           // token_parameters.value IN table.some_array
           // bucket.param IN table.some_array
@@ -371,7 +377,8 @@ export class SqlTools {
             usesUnauthenticatedRequestParameters: rightFilter.usesUnauthenticatedRequestParameters
           } satisfies ParameterMatchClause;
         } else {
-          return this.error(`Unsupported usage of IN operator`, expr);
+          // Not supported, return the error previously computed
+          return this.error(composeType.error!, composeType.errorExpr);
         }
       } else if (BASIC_OPERATORS.has(op)) {
         const fnImpl = getOperatorFunction(op);
@@ -634,36 +641,19 @@ export class SqlTools {
    * @returns a compiled function clause
    */
   composeFunction(fnImpl: SqlFunction, argClauses: CompiledClause[], debugArgExpressions: Expr[]): CompiledClause {
-    let argsType: 'static' | 'row' | 'param' = 'static';
-    for (let i = 0; i < argClauses.length; i++) {
-      const debugArg = debugArgExpressions[i];
-      const clause = argClauses[i];
-      if (isClauseError(clause)) {
-        // Return immediately on error
-        return clause;
-      } else if (isStaticValueClause(clause)) {
-        // argsType unchanged
-      } else if (isParameterValueClause(clause)) {
-        if (!this.supports_parameter_expressions) {
-          return this.error(`Cannot use bucket parameters in expressions`, debugArg);
-        }
-        if (argsType == 'static' || argsType == 'param') {
-          argsType = 'param';
-        } else {
-          return this.error(`Cannot use table values and parameters in the same clauses`, debugArg);
-        }
-      } else if (isRowValueClause(clause)) {
-        if (argsType == 'static' || argsType == 'row') {
-          argsType = 'row';
-        } else {
-          return this.error(`Cannot use table values and parameters in the same clauses`, debugArg);
-        }
-      } else {
-        return this.error(`Parameter match clauses cannot be used here`, debugArg);
-      }
+    const result = this.getComposeType(fnImpl, argClauses, debugArgExpressions);
+    if (result.errorClause != null) {
+      return result.errorClause;
+    } else if (result.error != null) {
+      return this.error(result.error, result.errorExpr);
     }
+    const argsType = result.argsType!;
 
-    if (argsType == 'row' || argsType == 'static') {
+    if (argsType == 'static') {
+      const args = argClauses.map((e) => (e as StaticValueClause).value);
+      const evaluated = fnImpl.call(...args);
+      return staticValueClause(evaluated);
+    } else if (argsType == 'row') {
       return {
         evaluate: (tables) => {
           const args = argClauses.map((e) => (e as RowValueClause).evaluate(tables));
@@ -705,7 +695,48 @@ export class SqlTools {
     }
   }
 
-  parameterFunction() {}
+  getComposeType(
+    fnImpl: SqlFunction,
+    argClauses: CompiledClause[],
+    debugArgExpressions: Expr[]
+  ): { argsType?: string; error?: string; errorExpr?: Expr; errorClause?: ClauseError } {
+    let argsType: 'static' | 'row' | 'param' = 'static';
+    for (let i = 0; i < argClauses.length; i++) {
+      const debugArg = debugArgExpressions[i];
+      const clause = argClauses[i];
+      if (isClauseError(clause)) {
+        // Return immediately on error
+        return { errorClause: clause };
+      } else if (isStaticValueClause(clause)) {
+        // argsType unchanged
+      } else if (isParameterValueClause(clause)) {
+        if (!this.supports_parameter_expressions) {
+          if (fnImpl.debugName == 'operatorIN') {
+            // Special-case error message to be more descriptive
+            return { error: `Cannot use bucket parameters on the right side of IN operators`, errorExpr: debugArg };
+          }
+          return { error: `Cannot use bucket parameters in expressions`, errorExpr: debugArg };
+        }
+        if (argsType == 'static' || argsType == 'param') {
+          argsType = 'param';
+        } else {
+          return { error: `Cannot use table values and parameters in the same clauses`, errorExpr: debugArg };
+        }
+      } else if (isRowValueClause(clause)) {
+        if (argsType == 'static' || argsType == 'row') {
+          argsType = 'row';
+        } else {
+          return { error: `Cannot use table values and parameters in the same clauses`, errorExpr: debugArg };
+        }
+      } else {
+        return { error: `Parameter match clauses cannot be used here`, errorExpr: debugArg };
+      }
+    }
+
+    return {
+      argsType
+    };
+  }
 }
 
 function isStatic(expr: Expr) {

packages/sync-rules/src/sql_functions.ts

@@ -1,5 +1,5 @@
 import { JSONBig } from '@powersync/service-jsonbig';
-import { SQLITE_FALSE, SQLITE_TRUE, sqliteBool, sqliteNot } from './sql_support.js';
+import { getOperatorFunction, SQLITE_FALSE, SQLITE_TRUE, sqliteBool, sqliteNot } from './sql_support.js';
 import { SqliteValue } from './types.js';
 import { jsonValueToSqlite } from './utils.js';
 // Declares @syncpoint/wkx module
@@ -787,6 +787,8 @@ export const OPERATOR_NOT: SqlFunction = {
   }
 };
 
+export const OPERATOR_IN = getOperatorFunction('IN');
+
 export function castOperator(castTo: string | undefined): SqlFunction | null {
   if (castTo == null || !CAST_TYPES.has(castTo)) {
     return null;

packages/sync-rules/test/src/data_queries.test.ts

@@ -41,6 +41,38 @@ describe('data queries', () => {
     expect(query.evaluateRow(ASSETS, { id: 'asset1', org_id: null })).toEqual([]);
   });
 
+  test('static IN data query', function () {
+    const sql = `SELECT * FROM assets WHERE 'green' IN assets.categories`;
+    const query = SqlDataQuery.fromSql('mybucket', [], sql);
+    expect(query.errors).toEqual([]);
+
+    expect(query.evaluateRow(ASSETS, { id: 'asset1', categories: JSON.stringify(['red', 'green']) })).toMatchObject([
+      {
+        bucket: 'mybucket[]',
+        table: 'assets',
+        id: 'asset1'
+      }
+    ]);
+
+    expect(query.evaluateRow(ASSETS, { id: 'asset1', categories: JSON.stringify(['red', 'blue']) })).toEqual([]);
+  });
+
+  test('data IN static query', function () {
+    const sql = `SELECT * FROM assets WHERE assets.condition IN '["good","great"]'`;
+    const query = SqlDataQuery.fromSql('mybucket', [], sql);
+    expect(query.errors).toEqual([]);
+
+    expect(query.evaluateRow(ASSETS, { id: 'asset1', condition: 'good' })).toMatchObject([
+      {
+        bucket: 'mybucket[]',
+        table: 'assets',
+        id: 'asset1'
+      }
+    ]);
+
+    expect(query.evaluateRow(ASSETS, { id: 'asset1', condition: 'bad' })).toEqual([]);
+  });
+
   test('table alias', function () {
     const sql = 'SELECT * FROM assets as others WHERE others.org_id = bucket.org_id';
     const query = SqlDataQuery.fromSql('mybucket', ['org_id'], sql);
@@ -158,7 +190,9 @@ describe('data queries', () => {
   test('invalid query - invalid IN', function () {
     const sql = 'SELECT * FROM assets WHERE assets.category IN bucket.categories';
     const query = SqlDataQuery.fromSql('mybucket', ['categories'], sql);
-    expect(query.errors).toMatchObject([{ type: 'fatal', message: 'Unsupported usage of IN operator' }]);
+    expect(query.errors).toMatchObject([
+      { type: 'fatal', message: 'Cannot use bucket parameters on the right side of IN operators' }
+    ]);
   });
 
   test('invalid query - not all parameters used', function () {

packages/sync-rules/test/src/static_parameter_queries.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, test } from 'vitest';
-import { SqlParameterQuery } from '../../src/index.js';
+import { RequestParameters, SqlParameterQuery } from '../../src/index.js';
 import { StaticSqlParameterQuery } from '../../src/StaticSqlParameterQuery.js';
 import { normalizeTokenParameters } from './util.js';
 
@@ -82,6 +82,68 @@ describe('static parameter queries', () => {
     expect(query.getStaticBucketIds(normalizeTokenParameters({ user_id: 'user1' }))).toEqual(['mybucket["user1"]']);
   });
 
+  test('static value', function () {
+    const sql = `SELECT WHERE 1`;
+    const query = SqlParameterQuery.fromSql('mybucket', sql) as StaticSqlParameterQuery;
+    expect(query.errors).toEqual([]);
+    expect(query.getStaticBucketIds(new RequestParameters({ sub: '' }, {}))).toEqual(['mybucket[]']);
+  });
+
+  test('static expression (1)', function () {
+    const sql = `SELECT WHERE 1 = 1`;
+    const query = SqlParameterQuery.fromSql('mybucket', sql) as StaticSqlParameterQuery;
+    expect(query.errors).toEqual([]);
+    expect(query.getStaticBucketIds(new RequestParameters({ sub: '' }, {}))).toEqual(['mybucket[]']);
+  });
+
+  test('static expression (2)', function () {
+    const sql = `SELECT WHERE 1 != 1`;
+    const query = SqlParameterQuery.fromSql('mybucket', sql) as StaticSqlParameterQuery;
+    expect(query.errors).toEqual([]);
+    expect(query.getStaticBucketIds(new RequestParameters({ sub: '' }, {}))).toEqual([]);
+  });
+
+  test('static IN expression', function () {
+    const sql = `SELECT WHERE 'admin' IN '["admin", "superuser"]'`;
+    const query = SqlParameterQuery.fromSql('mybucket', sql, undefined, {}) as StaticSqlParameterQuery;
+    expect(query.errors).toEqual([]);
+    expect(query.getStaticBucketIds(new RequestParameters({ sub: '' }, {}))).toEqual(['mybucket[]']);
+  });
+
+  test('IN for permissions in request.jwt() (1)', function () {
+    // Can use -> or ->> here
+    const sql = `SELECT 'read:users' IN (request.jwt() ->> 'permissions') as access_granted`;
+    const query = SqlParameterQuery.fromSql('mybucket', sql) as StaticSqlParameterQuery;
+    expect(query.errors).toEqual([]);
+    expect(
+      query.getStaticBucketIds(new RequestParameters({ sub: '', permissions: ['write', 'read:users'] }, {}))
+    ).toEqual(['mybucket[1]']);
+    expect(
+      query.getStaticBucketIds(new RequestParameters({ sub: '', permissions: ['write', 'write:users'] }, {}))
+    ).toEqual(['mybucket[0]']);
+  });
+
+  test('IN for permissions in request.jwt() (2)', function () {
+    // Can use -> or ->> here
+    const sql = `SELECT WHERE 'read:users' IN (request.jwt() ->> 'permissions')`;
+    const query = SqlParameterQuery.fromSql('mybucket', sql, undefined, {}) as StaticSqlParameterQuery;
+    expect(query.errors).toEqual([]);
+    expect(
+      query.getStaticBucketIds(new RequestParameters({ sub: '', permissions: ['write', 'read:users'] }, {}))
+    ).toEqual(['mybucket[]']);
+    expect(
+      query.getStaticBucketIds(new RequestParameters({ sub: '', permissions: ['write', 'write:users'] }, {}))
+    ).toEqual([]);
+  });
+
+  test('IN for permissions in request.jwt() (3)', function () {
+    const sql = `SELECT WHERE request.jwt() ->> 'role' IN '["admin", "superuser"]'`;
+    const query = SqlParameterQuery.fromSql('mybucket', sql, undefined, {}) as StaticSqlParameterQuery;
+    expect(query.errors).toEqual([]);
+    expect(query.getStaticBucketIds(new RequestParameters({ sub: '', role: 'superuser' }, {}))).toEqual(['mybucket[]']);
+    expect(query.getStaticBucketIds(new RequestParameters({ sub: '', role: 'superadmin' }, {}))).toEqual([]);
+  });
+
   test('case-sensitive queries (1)', () => {
     const sql = 'SELECT request.user_id() as USER_ID';
     const query = SqlParameterQuery.fromSql('mybucket', sql) as SqlParameterQuery;

service/README.md

@@ -2,7 +2,7 @@
   <a href="https://www.powersync.com" target="_blank"><img src="https://github.com/powersync-ja/.github/assets/7372448/d2538c43-c1a0-4c47-9a76-41462dba484f"/></a>
 </p>
 
-*[PowerSync](https://www.powersync.com) is a Postgres-SQLite sync engine, which helps developers to create local-first real-time reactive apps that work seamlessly both online and offline.*
+*[PowerSync](https://www.powersync.com) is a sync engine for building local-first apps with instantly-responsive UI/UX and simplified state transfer. Syncs between SQLite on the client-side and Postgres or MongoDB on the server-side (MySQL coming soon).*
 
 # Quick reference