From 7618e410bedf3e7cafa797a9db25359218f75e84 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Tue, 26 Nov 2024 11:13:44 +0200 Subject: [PATCH 1/4] remove postgres errors from mysql+mongodb rate limiters --- .../src/replication/MongoErrorRateLimiter.ts | 9 +-------- .../src/replication/MySQLErrorRateLimiter.ts | 7 ------- packages/service-core/src/routes/endpoints/sync-rules.ts | 3 +-- 3 files changed, 2 insertions(+), 17 deletions(-) diff --git a/modules/module-mongodb/src/replication/MongoErrorRateLimiter.ts b/modules/module-mongodb/src/replication/MongoErrorRateLimiter.ts index bcf58db6..17b65c66 100644 --- a/modules/module-mongodb/src/replication/MongoErrorRateLimiter.ts +++ b/modules/module-mongodb/src/replication/MongoErrorRateLimiter.ts @@ -1,5 +1,5 @@ -import { setTimeout } from 'timers/promises'; import { ErrorRateLimiter } from '@powersync/service-core'; +import { setTimeout } from 'timers/promises'; export class MongoErrorRateLimiter implements ErrorRateLimiter { nextAllowed: number = Date.now(); @@ -27,13 +27,6 @@ export class MongoErrorRateLimiter implements ErrorRateLimiter { } else if (message.includes('ECONNREFUSED')) { // Could be fail2ban or similar this.setDelay(120_000); - } else if ( - message.includes('Unable to do postgres query on ended pool') || - message.includes('Postgres unexpectedly closed connection') - ) { - // Connection timed out - ignore / immediately retry - // We don't explicitly set the delay to 0, since there could have been another error that - // we need to respect. } else { this.setDelay(30_000); } diff --git a/modules/module-mysql/src/replication/MySQLErrorRateLimiter.ts b/modules/module-mysql/src/replication/MySQLErrorRateLimiter.ts index aff96c3a..8966cd20 100644 --- a/modules/module-mysql/src/replication/MySQLErrorRateLimiter.ts +++ b/modules/module-mysql/src/replication/MySQLErrorRateLimiter.ts @@ -26,13 +26,6 @@ export class MySQLErrorRateLimiter implements ErrorRateLimiter { } else if (message.includes('ECONNREFUSED')) { // Could be fail2ban or similar this.setDelay(120_000); - } else if ( - message.includes('Unable to do postgres query on ended pool') || - message.includes('Postgres unexpectedly closed connection') - ) { - // Connection timed out - ignore / immediately retry - // We don't explicitly set the delay to 0, since there could have been another error that - // we need to respect. } else { this.setDelay(30_000); } diff --git a/packages/service-core/src/routes/endpoints/sync-rules.ts b/packages/service-core/src/routes/endpoints/sync-rules.ts index 70e6127d..c902171c 100644 --- a/packages/service-core/src/routes/endpoints/sync-rules.ts +++ b/packages/service-core/src/routes/endpoints/sync-rules.ts @@ -3,10 +3,9 @@ import { SqlSyncRules, SyncRulesErrors } from '@powersync/service-sync-rules'; import type { FastifyPluginAsync } from 'fastify'; import * as t from 'ts-codec'; -import * as system from '../../system/system-index.js'; +import { RouteAPI } from '../../api/RouteAPI.js'; import { authApi } from '../auth.js'; import { routeDefinition } from '../router.js'; -import { RouteAPI } from '../../api/RouteAPI.js'; const DeploySyncRulesRequest = t.object({ content: t.string From 647df7fe6089690cfa7604e8dff2dfd9a6d8e19c Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Tue, 26 Nov 2024 11:43:12 +0200 Subject: [PATCH 2/4] use common typescript version --- modules/module-mongodb/package.json | 5 +-- modules/module-mysql/package.json | 5 +-- modules/module-postgres/package.json | 5 +-- package.json | 2 + packages/rsocket-router/package.json | 4 +- packages/service-core/package.json | 5 +-- pnpm-lock.yaml | 67 +++++----------------------- service/package.json | 4 +- test-client/package.json | 3 +- 9 files changed, 20 insertions(+), 80 deletions(-) diff --git a/modules/module-mongodb/package.json b/modules/module-mongodb/package.json index d928ec51..8236e3b2 100644 --- a/modules/module-mongodb/package.json +++ b/modules/module-mongodb/package.json @@ -39,9 +39,6 @@ "uri-js": "^4.4.1" }, "devDependencies": { - "@types/uuid": "^9.0.4", - "typescript": "^5.2.2", - "vitest": "^2.1.1", - "vite-tsconfig-paths": "^4.3.2" + "@types/uuid": "^9.0.4" } } diff --git a/modules/module-mysql/package.json b/modules/module-mysql/package.json index 18611a52..69cb93a6 100644 --- a/modules/module-mysql/package.json +++ b/modules/module-mysql/package.json @@ -44,9 +44,6 @@ "devDependencies": { "@types/semver": "^7.5.4", "@types/async": "^3.2.24", - "@types/uuid": "^9.0.4", - "typescript": "^5.5.4", - "vite-tsconfig-paths": "^4.3.2", - "vitest": "^2.1.1" + "@types/uuid": "^9.0.4" } } diff --git a/modules/module-postgres/package.json b/modules/module-postgres/package.json index d42db920..b34b8dd8 100644 --- a/modules/module-postgres/package.json +++ b/modules/module-postgres/package.json @@ -41,9 +41,6 @@ "uri-js": "^4.4.1" }, "devDependencies": { - "@types/uuid": "^9.0.4", - "typescript": "^5.6.2", - "vitest": "^2.1.1", - "vite-tsconfig-paths": "^4.3.2" + "@types/uuid": "^9.0.4" } } diff --git a/package.json b/package.json index 0f8fe506..1846299d 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,8 @@ "tsc-watch": "^6.2.0", "ts-node-dev": "^2.0.0", "typescript": "^5.6.2", + "vite-tsconfig-paths": "^4.3.2", + "vitest": "^2.1.1", "ws": "^8.2.3" } } diff --git a/packages/rsocket-router/package.json b/packages/rsocket-router/package.json index 97747a9f..039abc5d 100644 --- a/packages/rsocket-router/package.json +++ b/packages/rsocket-router/package.json @@ -28,8 +28,6 @@ "@types/uuid": "^9.0.4", "@types/ws": "~8.2.0", "bson": "^6.6.0", - "rsocket-websocket-client": "1.0.0-alpha.3", - "typescript": "^5.6.2", - "vitest": "^2.1.1" + "rsocket-websocket-client": "1.0.0-alpha.3" } } diff --git a/packages/service-core/package.json b/packages/service-core/package.json index 54ee47f2..df1e5976 100644 --- a/packages/service-core/package.json +++ b/packages/service-core/package.json @@ -50,9 +50,6 @@ "@types/lodash": "^4.17.5", "@types/uuid": "^9.0.4", "fastify": "4.23.2", - "fastify-plugin": "^4.5.1", - "typescript": "^5.6.2", - "vite-tsconfig-paths": "^4.3.2", - "vitest": "^2.1.1" + "fastify-plugin": "^4.5.1" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index cd892d26..9ce1d011 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -50,6 +50,12 @@ importers: typescript: specifier: ^5.6.2 version: 5.6.2 + vite-tsconfig-paths: + specifier: ^4.3.2 + version: 4.3.2(typescript@5.6.2)(vite@5.3.3(@types/node@22.5.5)) + vitest: + specifier: ^2.1.1 + version: 2.1.1(@types/node@22.5.5) ws: specifier: ^8.2.3 version: 8.18.0 @@ -127,15 +133,6 @@ importers: '@types/uuid': specifier: ^9.0.4 version: 9.0.8 - typescript: - specifier: ^5.2.2 - version: 5.6.2 - vite-tsconfig-paths: - specifier: ^4.3.2 - version: 4.3.2(typescript@5.6.2)(vite@5.3.3(@types/node@22.5.5)) - vitest: - specifier: ^2.1.1 - version: 2.1.1(@types/node@22.5.5) modules/module-mysql: dependencies: @@ -185,15 +182,6 @@ importers: '@types/uuid': specifier: ^9.0.4 version: 9.0.8 - typescript: - specifier: ^5.5.4 - version: 5.6.2 - vite-tsconfig-paths: - specifier: ^4.3.2 - version: 4.3.2(typescript@5.6.2)(vite@5.3.3(@types/node@22.5.5)) - vitest: - specifier: ^2.1.1 - version: 2.1.1(@types/node@22.5.5) modules/module-postgres: dependencies: @@ -234,15 +222,6 @@ importers: '@types/uuid': specifier: ^9.0.4 version: 9.0.8 - typescript: - specifier: ^5.6.2 - version: 5.6.2 - vite-tsconfig-paths: - specifier: ^4.3.2 - version: 4.3.2(typescript@5.6.2)(vite@5.3.3(@types/node@22.5.5)) - vitest: - specifier: ^2.1.1 - version: 2.1.1(@types/node@22.5.5) packages/jpgwire: dependencies: @@ -298,12 +277,6 @@ importers: rsocket-websocket-client: specifier: 1.0.0-alpha.3 version: 1.0.0-alpha.3 - typescript: - specifier: ^5.6.2 - version: 5.6.2 - vitest: - specifier: ^2.1.1 - version: 2.1.1(@types/node@22.5.5) packages/service-core: dependencies: @@ -407,15 +380,6 @@ importers: fastify-plugin: specifier: ^4.5.1 version: 4.5.1 - typescript: - specifier: ^5.6.2 - version: 5.6.2 - vite-tsconfig-paths: - specifier: ^4.3.2 - version: 4.3.2(typescript@5.6.2)(vite@5.3.3(@types/node@22.5.5)) - vitest: - specifier: ^2.1.1 - version: 2.1.1(@types/node@22.5.5) packages/sync-rules: dependencies: @@ -565,12 +529,6 @@ importers: ts-node: specifier: ^10.9.1 version: 10.9.2(@types/node@22.5.5)(typescript@5.6.2) - typescript: - specifier: ^5.6.2 - version: 5.6.2 - vitest: - specifier: ^2.1.1 - version: 2.1.1(@types/node@22.5.5) test-client: dependencies: @@ -590,9 +548,6 @@ importers: '@types/node': specifier: ^22.5.5 version: 22.5.5 - typescript: - specifier: ^5.6.2 - version: 5.6.2 packages: @@ -4675,7 +4630,7 @@ snapshots: '@opentelemetry/semantic-conventions': 1.25.1 '@prisma/instrumentation': 5.16.1 '@sentry/core': 8.17.0 - '@sentry/opentelemetry': 8.17.0(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.25.1(@opentelemetry/api@1.6.0))(@opentelemetry/instrumentation@0.52.1(@opentelemetry/api@1.6.0))(@opentelemetry/sdk-trace-base@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/semantic-conventions@1.25.1) + '@sentry/opentelemetry': 8.17.0(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.25.1(@opentelemetry/api@1.6.0))(@opentelemetry/instrumentation@0.52.1(@opentelemetry/api@1.6.0))(@opentelemetry/sdk-trace-base@1.25.1(@opentelemetry/api@1.6.0))(@opentelemetry/semantic-conventions@1.25.1) '@sentry/types': 8.17.0 '@sentry/utils': 8.17.0 optionalDependencies: @@ -4683,7 +4638,7 @@ snapshots: transitivePeerDependencies: - supports-color - '@sentry/opentelemetry@8.17.0(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.25.1(@opentelemetry/api@1.6.0))(@opentelemetry/instrumentation@0.52.1(@opentelemetry/api@1.6.0))(@opentelemetry/sdk-trace-base@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/semantic-conventions@1.25.1)': + '@sentry/opentelemetry@8.17.0(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.25.1(@opentelemetry/api@1.6.0))(@opentelemetry/instrumentation@0.52.1(@opentelemetry/api@1.6.0))(@opentelemetry/sdk-trace-base@1.25.1(@opentelemetry/api@1.6.0))(@opentelemetry/semantic-conventions@1.25.1)': dependencies: '@opentelemetry/api': 1.9.0 '@opentelemetry/core': 1.25.1(@opentelemetry/api@1.9.0) @@ -4880,7 +4835,7 @@ snapshots: agent-base@6.0.2: dependencies: - debug: 4.3.5(supports-color@5.5.0) + debug: 4.3.7 transitivePeerDependencies: - supports-color @@ -7044,7 +6999,7 @@ snapshots: tuf-js@1.1.7: dependencies: '@tufjs/models': 1.0.4 - debug: 4.3.5(supports-color@5.5.0) + debug: 4.3.7 make-fetch-happen: 11.1.1 transitivePeerDependencies: - supports-color @@ -7143,7 +7098,7 @@ snapshots: vite-tsconfig-paths@4.3.2(typescript@5.6.2)(vite@5.3.3(@types/node@22.5.5)): dependencies: - debug: 4.3.5(supports-color@5.5.0) + debug: 4.3.7 globrex: 0.1.2 tsconfck: 3.1.1(typescript@5.6.2) optionalDependencies: diff --git a/service/package.json b/service/package.json index e54948ba..10f636fa 100644 --- a/service/package.json +++ b/service/package.json @@ -48,8 +48,6 @@ "copyfiles": "^2.4.1", "nodemon": "^3.0.1", "npm-check-updates": "^16.14.4", - "ts-node": "^10.9.1", - "typescript": "^5.6.2", - "vitest": "^2.1.1" + "ts-node": "^10.9.1" } } diff --git a/test-client/package.json b/test-client/package.json index ddf8e5e6..4d5f3d9d 100644 --- a/test-client/package.json +++ b/test-client/package.json @@ -20,7 +20,6 @@ "yaml": "^2.5.0" }, "devDependencies": { - "@types/node": "^22.5.5", - "typescript": "^5.6.2" + "@types/node": "^22.5.5" } } From df0d5d7a9f725730e86f3d430858532615f7c8c7 Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Tue, 26 Nov 2024 11:55:00 +0200 Subject: [PATCH 3/4] update changeset --- .changeset/tender-vans-impress.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/tender-vans-impress.md b/.changeset/tender-vans-impress.md index 0da1c484..10696036 100644 --- a/.changeset/tender-vans-impress.md +++ b/.changeset/tender-vans-impress.md @@ -4,7 +4,7 @@ '@powersync/lib-services-framework': minor '@powersync/service-jpgwire': minor '@powersync/service-types': minor -'@powersync/service-image': minor +'@powersync/service-image': major '@powersync/service-module-postgres': patch --- From 5d24dd106f1ebd093e0b96345abb25cebe78e05f Mon Sep 17 00:00:00 2001 From: stevensJourney Date: Tue, 26 Nov 2024 16:35:29 +0200 Subject: [PATCH 4/4] update zongji package --- modules/module-mysql/package.json | 2 +- pnpm-lock.yaml | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/module-mysql/package.json b/modules/module-mysql/package.json index 69cb93a6..584ff1fa 100644 --- a/modules/module-mysql/package.json +++ b/modules/module-mysql/package.json @@ -33,7 +33,7 @@ "@powersync/service-sync-rules": "workspace:*", "@powersync/service-types": "workspace:*", "@powersync/service-jsonbig": "workspace:*", - "@powersync/mysql-zongji": "0.0.0-dev-20241031142605", + "@powersync/mysql-zongji": "^0.1.0", "semver": "^7.5.4", "async": "^3.2.4", "mysql2": "^3.11.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9ce1d011..012efe2c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -140,8 +140,8 @@ importers: specifier: workspace:* version: link:../../libs/lib-services '@powersync/mysql-zongji': - specifier: 0.0.0-dev-20241031142605 - version: 0.0.0-dev-20241031142605 + specifier: ^0.1.0 + version: 0.1.0 '@powersync/service-core': specifier: workspace:* version: link:../../packages/service-core @@ -1136,8 +1136,8 @@ packages: resolution: {integrity: sha512-UA91GwWPhFExt3IizW6bOeY/pQ0BkuNwKjk9iQW9KqxluGCrg4VenZ0/L+2Y0+ZOtme72EVvg6v0zo3AMQRCeA==} engines: {node: '>=12'} - '@powersync/mysql-zongji@0.0.0-dev-20241031142605': - resolution: {integrity: sha512-LATx5xfJjXcbvW3s9yCy2tlhhsgOuaUoUk0/EH67sgbTQQFZTfBVO/2gzdJH5GvGgONdqoPBQ7skWNwBF/dEXw==} + '@powersync/mysql-zongji@0.1.0': + resolution: {integrity: sha512-2GjOxVws+wtbb+xFUJe4Ozzkp/f0Gsna0fje9art76bmz6yfLCW4K3Mf2/M310xMnAIp8eP9hsJ6DYwwZCo1RA==} engines: {node: '>=20.0.0'} '@prisma/instrumentation@5.16.1': @@ -4537,7 +4537,7 @@ snapshots: '@pnpm/network.ca-file': 1.0.2 config-chain: 1.1.13 - '@powersync/mysql-zongji@0.0.0-dev-20241031142605': + '@powersync/mysql-zongji@0.1.0': dependencies: '@vlasky/mysql': 2.18.6 big-integer: 1.6.51