Skip to content

Commit 43a10fa

Browse files
rkistnerrkistner
authored
Merge pull request #90 from powersync-ja/generic-schema-definitions
Use generic schema definitions
2 parents 417cf5f + 4ecaee2 commit 43a10fa

File tree

15 files changed

+378
-195
lines changed

15 files changed

+378
-195
lines changed

‎.changeset/weak-cats-hug.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
'@powersync/service-sync-rules': minor
3+
---
4+
5+
Optionally include original types in generated schemas as a comment.

‎modules/module-postgres/src/api/PostgresRouteAPIAdapter.ts

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -275,14 +275,14 @@ GROUP BY schemaname, tablename, quoted_name`
275275
);
276276
const rows = pgwire.pgwireRows(results);
277277

278-
let schemas: Record<string, any> = {};
278+
let schemas: Record<string, service_types.DatabaseSchema> = {};
279279

280280
for (let row of rows) {
281281
const schema = (schemas[row.schemaname] ??= {
282282
name: row.schemaname,
283283
tables: []
284284
});
285-
const table = {
285+
const table: service_types.TableSchema = {
286286
name: row.tablename,
287287
columns: [] as any[]
288288
};
@@ -296,7 +296,9 @@ GROUP BY schemaname, tablename, quoted_name`
296296
}
297297
table.columns.push({
298298
name: column.attname,
299+
sqlite_type: sync_rules.expressionTypeFromPostgresType(pg_type).typeFlags,
299300
type: column.data_type,
301+
internal_type: column.data_type,
300302
pg_type: pg_type
301303
});
302304
}

‎packages/sync-rules/src/DartSchemaGenerator.ts

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import { ColumnDefinition, TYPE_INTEGER, TYPE_REAL, TYPE_TEXT } from './ExpressionType.js';
2-
import { SchemaGenerator } from './SchemaGenerator.js';
2+
import { GenerateSchemaOptions, SchemaGenerator } from './SchemaGenerator.js';
33
import { SqlSyncRules } from './SqlSyncRules.js';
44
import { SourceSchema } from './types.js';
55

@@ -9,18 +9,34 @@ export class DartSchemaGenerator extends SchemaGenerator {
99
readonly mediaType = 'text/x-dart';
1010
readonly fileName = 'schema.dart';
1111

12-
generate(source: SqlSyncRules, schema: SourceSchema): string {
12+
generate(source: SqlSyncRules, schema: SourceSchema, options?: GenerateSchemaOptions): string {
1313
const tables = super.getAllTables(source, schema);
1414

1515
return `Schema([
16-
${tables.map((table) => this.generateTable(table.name, table.columns)).join(',\n ')}
16+
${tables.map((table) => this.generateTable(table.name, table.columns, options)).join(',\n ')}
1717
]);
1818
`;
1919
}
2020

21-
private generateTable(name: string, columns: ColumnDefinition[]): string {
21+
private generateTable(name: string, columns: ColumnDefinition[], options?: GenerateSchemaOptions): string {
22+
const generated = columns.map((c, i) => {
23+
const last = i == columns.length - 1;
24+
const base = this.generateColumn(c);
25+
let withFormatting: string;
26+
if (last) {
27+
withFormatting = ` ${base}`;
28+
} else {
29+
withFormatting = ` ${base},`;
30+
}
31+
32+
if (options?.includeTypeComments && c.originalType != null) {
33+
return `${withFormatting} // ${c.originalType}`;
34+
} else {
35+
return withFormatting;
36+
}
37+
});
2238
return `Table('${name}', [
23-
${columns.map((c) => this.generateColumn(c)).join(',\n ')}
39+
${generated.join('\n')}
2440
])`;
2541
}
2642

‎packages/sync-rules/src/ExpressionType.ts

Lines changed: 28 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,12 @@ export const TYPE_TEXT = 2;
44
export const TYPE_INTEGER = 4;
55
export const TYPE_REAL = 8;
66

7-
export type SqliteType = 'null' | 'blob' | 'text' | 'integer' | 'real';
7+
export type SqliteType = 'null' | 'blob' | 'text' | 'integer' | 'real' | 'numeric';
88

99
export interface ColumnDefinition {
1010
name: string;
1111
type: ExpressionType;
12+
originalType?: string;
1213
}
1314

1415
export class ExpressionType {
@@ -34,7 +35,7 @@ export class ExpressionType {
3435
return new ExpressionType(typeFlags);
3536
}
3637

37-
static fromTypeText(type: SqliteType | 'numeric') {
38+
static fromTypeText(type: SqliteType) {
3839
if (type == 'null') {
3940
return ExpressionType.NONE;
4041
} else if (type == 'blob') {
@@ -72,3 +73,28 @@ export class ExpressionType {
7273
return this.typeFlags == TYPE_NONE;
7374
}
7475
}
76+
77+
/**
78+
* Here only for backwards-compatibility only.
79+
*/
80+
export function expressionTypeFromPostgresType(type: string | undefined): ExpressionType {
81+
if (type?.endsWith('[]')) {
82+
return ExpressionType.TEXT;
83+
}
84+
switch (type) {
85+
case 'bool':
86+
return ExpressionType.INTEGER;
87+
case 'bytea':
88+
return ExpressionType.BLOB;
89+
case 'int2':
90+
case 'int4':
91+
case 'int8':
92+
case 'oid':
93+
return ExpressionType.INTEGER;
94+
case 'float4':
95+
case 'float8':
96+
return ExpressionType.REAL;
97+
default:
98+
return ExpressionType.TEXT;
99+
}
100+
}

‎packages/sync-rules/src/SchemaGenerator.ts

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,10 @@ import { ColumnDefinition } from './ExpressionType.js';
22
import { SqlSyncRules } from './SqlSyncRules.js';
33
import { SourceSchema } from './types.js';
44

5+
export interface GenerateSchemaOptions {
6+
includeTypeComments?: boolean;
7+
}
8+
59
export abstract class SchemaGenerator {
610
protected getAllTables(source: SqlSyncRules, schema: SourceSchema) {
711
let tables: Record<string, Record<string, ColumnDefinition>> = {};
@@ -33,5 +37,5 @@ export abstract class SchemaGenerator {
3337
abstract readonly mediaType: string;
3438
abstract readonly fileName: string;
3539

36-
abstract generate(source: SqlSyncRules, schema: SourceSchema): string;
40+
abstract generate(source: SqlSyncRules, schema: SourceSchema, options?: GenerateSchemaOptions): string;
3741
}

‎packages/sync-rules/src/SqlDataQuery.ts

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,9 @@ export class SqlDataQuery {
123123
output[name] = clause.evaluate(tables);
124124
},
125125
getTypes(schema, into) {
126-
into[name] = { name, type: clause.getType(schema) };
126+
const def = clause.getColumnDefinition(schema);
127+
128+
into[name] = { name, type: def?.type ?? ExpressionType.NONE, originalType: def?.originalType };
127129
}
128130
});
129131
} else {
@@ -152,7 +154,7 @@ export class SqlDataQuery {
152154
// Not performing schema-based validation - assume there is an id
153155
hasId = true;
154156
} else {
155-
const idType = querySchema.getType(alias, 'id');
157+
const idType = querySchema.getColumn(alias, 'id')?.type ?? ExpressionType.NONE;
156158
if (!idType.isNone()) {
157159
hasId = true;
158160
}
@@ -296,12 +298,12 @@ export class SqlDataQuery {
296298

297299
private getColumnOutputsFor(schemaTable: SourceSchemaTable, output: Record<string, ColumnDefinition>) {
298300
const querySchema: QuerySchema = {
299-
getType: (table, column) => {
301+
getColumn: (table, column) => {
300302
if (table == this.table!) {
301-
return schemaTable.getType(column) ?? ExpressionType.NONE;
303+
return schemaTable.getColumn(column);
302304
} else {
303305
// TODO: bucket parameters?
304-
return ExpressionType.NONE;
306+
return undefined;
305307
}
306308
},
307309
getColumns: (table) => {

‎packages/sync-rules/src/StaticSchema.ts

Lines changed: 34 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { ColumnDefinition, ExpressionType } from './ExpressionType.js';
1+
import { ColumnDefinition, ExpressionType, expressionTypeFromPostgresType, SqliteType } from './ExpressionType.js';
22
import { SourceTableInterface } from './SourceTableInterface.js';
33
import { TablePattern } from './TablePattern.js';
44
import { SourceSchema, SourceSchemaTable } from './types.js';
@@ -14,11 +14,28 @@ export interface SourceTableDefinition {
1414
}
1515

1616
export interface SourceColumnDefinition {
17+
/**
18+
* Column name.
19+
*/
1720
name: string;
21+
22+
/**
23+
* Option 1: SQLite type flags - see ExpressionType.typeFlags.
24+
* Option 2: SQLite type name in lowercase - 'text' | 'integer' | 'real' | 'numeric' | 'blob' | 'null'
25+
*/
26+
sqlite_type?: number | SqliteType;
27+
1828
/**
19-
* Postgres type.
29+
* Type name from the source database, e.g. "character varying(255)[]"
2030
*/
21-
pg_type: string;
31+
internal_type?: string;
32+
33+
/**
34+
* Postgres type, kept for backwards-compatibility.
35+
*
36+
* @deprecated - use internal_type instead
37+
*/
38+
pg_type?: string;
2239
}
2340

2441
export interface SourceConnectionDefinition {
@@ -43,8 +60,8 @@ class SourceTableDetails implements SourceTableInterface, SourceSchemaTable {
4360
);
4461
}
4562

46-
getType(column: string): ExpressionType | undefined {
47-
return this.columns[column]?.type;
63+
getColumn(column: string): ColumnDefinition | undefined {
64+
return this.columns[column];
4865
}
4966

5067
getColumns(): ColumnDefinition[] {
@@ -75,28 +92,20 @@ export class StaticSchema implements SourceSchema {
7592
function mapColumn(column: SourceColumnDefinition): ColumnDefinition {
7693
return {
7794
name: column.name,
78-
type: mapType(column.pg_type)
95+
type: mapColumnType(column),
96+
originalType: column.internal_type
7997
};
8098
}
8199

82-
function mapType(type: string | undefined): ExpressionType {
83-
if (type?.endsWith('[]')) {
84-
return ExpressionType.TEXT;
85-
}
86-
switch (type) {
87-
case 'bool':
88-
return ExpressionType.INTEGER;
89-
case 'bytea':
90-
return ExpressionType.BLOB;
91-
case 'int2':
92-
case 'int4':
93-
case 'int8':
94-
case 'oid':
95-
return ExpressionType.INTEGER;
96-
case 'float4':
97-
case 'float8':
98-
return ExpressionType.REAL;
99-
default:
100-
return ExpressionType.TEXT;
100+
function mapColumnType(column: SourceColumnDefinition): ExpressionType {
101+
if (typeof column.sqlite_type == 'number') {
102+
return ExpressionType.of(column.sqlite_type);
103+
} else if (typeof column.sqlite_type == 'string') {
104+
return ExpressionType.fromTypeText(column.sqlite_type);
105+
} else if (column.pg_type != null) {
106+
// We still handle these types for backwards-compatibility of old schemas
107+
return expressionTypeFromPostgresType(column.pg_type);
108+
} else {
109+
throw new Error(`Cannot determine SQLite type of ${JSON.stringify(column)}`);
101110
}
102111
}

‎packages/sync-rules/src/TableQuerySchema.ts

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,20 @@
1-
import { ColumnDefinition, ExpressionType } from './ExpressionType.js';
1+
import { ColumnDefinition } from './ExpressionType.js';
22
import { QuerySchema, SourceSchemaTable } from './types.js';
33

44
export class TableQuerySchema implements QuerySchema {
5-
constructor(
6-
private tables: SourceSchemaTable[],
7-
private alias: string
8-
) {}
5+
constructor(private tables: SourceSchemaTable[], private alias: string) {}
96

10-
getType(table: string, column: string): ExpressionType {
7+
getColumn(table: string, column: string): ColumnDefinition | undefined {
118
if (table != this.alias) {
12-
return ExpressionType.NONE;
9+
return undefined;
1310
}
1411
for (let table of this.tables) {
15-
const t = table.getType(column);
12+
const t = table.getColumn(column);
1613
if (t != null) {
1714
return t;
1815
}
1916
}
20-
return ExpressionType.NONE;
17+
return undefined;
2118
}
2219

2320
getColumns(table: string): ColumnDefinition[] {

‎packages/sync-rules/src/TsSchemaGenerator.ts

Lines changed: 22 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import { ColumnDefinition, TYPE_INTEGER, TYPE_REAL, TYPE_TEXT } from './ExpressionType.js';
2-
import { SchemaGenerator } from './SchemaGenerator.js';
2+
import { GenerateSchemaOptions, SchemaGenerator } from './SchemaGenerator.js';
33
import { SqlSyncRules } from './SqlSyncRules.js';
44
import { SourceSchema } from './types.js';
55

@@ -47,12 +47,12 @@ export class TsSchemaGenerator extends SchemaGenerator {
4747
}
4848
}
4949

50-
generate(source: SqlSyncRules, schema: SourceSchema): string {
50+
generate(source: SqlSyncRules, schema: SourceSchema, options?: GenerateSchemaOptions): string {
5151
const tables = super.getAllTables(source, schema);
5252

5353
return `${this.generateImports()}
5454
55-
${tables.map((table) => this.generateTable(table.name, table.columns)).join('\n\n')}
55+
${tables.map((table) => this.generateTable(table.name, table.columns, options)).join('\n\n')}
5656
5757
export const AppSchema = new Schema({
5858
${tables.map((table) => table.name).join(',\n ')}
@@ -81,11 +81,28 @@ ${this.generateTypeExports()}`;
8181
}
8282
}
8383

84-
private generateTable(name: string, columns: ColumnDefinition[]): string {
84+
private generateTable(name: string, columns: ColumnDefinition[], options?: GenerateSchemaOptions): string {
85+
const generated = columns.map((c, i) => {
86+
const last = i == columns.length - 1;
87+
const base = this.generateColumn(c);
88+
let withFormatting: string;
89+
if (last) {
90+
withFormatting = ` ${base}`;
91+
} else {
92+
withFormatting = ` ${base},`;
93+
}
94+
95+
if (options?.includeTypeComments && c.originalType != null) {
96+
return `${withFormatting} // ${c.originalType}`;
97+
} else {
98+
return withFormatting;
99+
}
100+
});
101+
85102
return `const ${name} = new Table(
86103
{
87104
// id column (text) is automatically included
88-
${columns.map((c) => this.generateColumn(c)).join(',\n ')}
105+
${generated.join('\n')}
89106
},
90107
{ indexes: {} }
91108
);`;

0 commit comments

Comments
 (0)