Skip to content

Raw tables #300

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 18 commits into from
Jul 17, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import 'dart:async';
import 'dart:convert';
import 'dart:isolate';
import 'package:meta/meta.dart';

Expand Down Expand Up @@ -83,8 +84,12 @@ class PowerSyncDatabaseImpl
DefaultSqliteOpenFactory factory =
// ignore: deprecated_member_use_from_same_package
PowerSyncOpenFactory(path: path, sqliteSetup: sqliteSetup);
return PowerSyncDatabaseImpl.withFactory(factory,
schema: schema, maxReaders: maxReaders, logger: logger);
return PowerSyncDatabaseImpl.withFactory(
factory,
schema: schema,
maxReaders: maxReaders,
logger: logger,
);
}

/// Open a [PowerSyncDatabase] with a [PowerSyncOpenFactory].
Expand All @@ -96,22 +101,29 @@ class PowerSyncDatabaseImpl
///
/// [logger] defaults to [autoLogger], which logs to the console in debug builds.
factory PowerSyncDatabaseImpl.withFactory(
DefaultSqliteOpenFactory openFactory,
{required Schema schema,
int maxReaders = SqliteDatabase.defaultMaxReaders,
Logger? logger}) {
DefaultSqliteOpenFactory openFactory, {
required Schema schema,
int maxReaders = SqliteDatabase.defaultMaxReaders,
Logger? logger,
}) {
final db = SqliteDatabase.withFactory(openFactory, maxReaders: maxReaders);
return PowerSyncDatabaseImpl.withDatabase(
schema: schema, database: db, logger: logger);
schema: schema,
database: db,
logger: logger,
);
}

/// Open a PowerSyncDatabase on an existing [SqliteDatabase].
///
/// Migrations are run on the database when this constructor is called.
///
/// [logger] defaults to [autoLogger], which logs to the console in debug builds.s
PowerSyncDatabaseImpl.withDatabase(
{required this.schema, required this.database, Logger? logger}) {
PowerSyncDatabaseImpl.withDatabase({
required this.schema,
required this.database,
Logger? logger,
}) {
this.logger = logger ?? autoLogger;
isInitialized = baseInit();
}
Expand Down Expand Up @@ -247,6 +259,7 @@ class PowerSyncDatabaseImpl
options,
crudMutex.shared,
syncMutex.shared,
jsonEncode(schema),
),
debugName: 'Sync ${database.openFactory.path}',
onError: receiveUnhandledErrors.sendPort,
Expand Down Expand Up @@ -290,13 +303,15 @@ class _PowerSyncDatabaseIsolateArgs {
final ResolvedSyncOptions options;
final SerializedMutex crudMutex;
final SerializedMutex syncMutex;
final String schemaJson;

_PowerSyncDatabaseIsolateArgs(
this.sPort,
this.dbRef,
this.options,
this.crudMutex,
this.syncMutex,
this.schemaJson,
);
}

Expand Down Expand Up @@ -392,6 +407,7 @@ Future<void> _syncIsolate(_PowerSyncDatabaseIsolateArgs args) async {
final storage = BucketStorage(connection);
final sync = StreamingSyncImplementation(
adapter: storage,
schemaJson: args.schemaJson,
connector: InternalConnector(
getCredentialsCached: getCredentialsCached,
prefetchCredentials: prefetchCredentials,
Expand Down
44 changes: 26 additions & 18 deletions packages/powersync_core/lib/src/database/powersync_database.dart
Original file line number Diff line number Diff line change
Expand Up @@ -32,19 +32,21 @@ abstract class PowerSyncDatabase
/// A maximum of [maxReaders] concurrent read transactions are allowed.
///
/// [logger] defaults to [autoLogger], which logs to the console in debug builds.
factory PowerSyncDatabase(
{required Schema schema,
required String path,
Logger? logger,
@Deprecated("Use [PowerSyncDatabase.withFactory] instead.")
// ignore: deprecated_member_use_from_same_package
SqliteConnectionSetup? sqliteSetup}) {
factory PowerSyncDatabase({
required Schema schema,
required String path,
Logger? logger,
@Deprecated("Use [PowerSyncDatabase.withFactory] instead.")
// ignore: deprecated_member_use_from_same_package
SqliteConnectionSetup? sqliteSetup,
}) {
return PowerSyncDatabaseImpl(
schema: schema,
path: path,
logger: logger,
// ignore: deprecated_member_use_from_same_package
sqliteSetup: sqliteSetup);
schema: schema,
path: path,
logger: logger,
// ignore: deprecated_member_use_from_same_package
sqliteSetup: sqliteSetup,
);
}

/// Open a [PowerSyncDatabase] with a [PowerSyncOpenFactory].
Expand All @@ -55,12 +57,18 @@ abstract class PowerSyncDatabase
/// Subclass [PowerSyncOpenFactory] to add custom logic to this process.
///
/// [logger] defaults to [autoLogger], which logs to the console in debug builds.
factory PowerSyncDatabase.withFactory(DefaultSqliteOpenFactory openFactory,
{required Schema schema,
int maxReaders = SqliteDatabase.defaultMaxReaders,
Logger? logger}) {
return PowerSyncDatabaseImpl.withFactory(openFactory,
schema: schema, maxReaders: maxReaders, logger: logger);
factory PowerSyncDatabase.withFactory(
DefaultSqliteOpenFactory openFactory, {
required Schema schema,
int maxReaders = SqliteDatabase.defaultMaxReaders,
Logger? logger,
}) {
return PowerSyncDatabaseImpl.withFactory(
openFactory,
schema: schema,
maxReaders: maxReaders,
logger: logger,
);
}

/// Open a PowerSyncDatabase on an existing [SqliteDatabase].
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -82,10 +82,11 @@ class PowerSyncDatabaseImpl
/// Migrations are run on the database when this constructor is called.
///
/// [logger] defaults to [autoLogger], which logs to the console in debug builds.s
factory PowerSyncDatabaseImpl.withDatabase(
{required Schema schema,
required SqliteDatabase database,
Logger? logger}) {
factory PowerSyncDatabaseImpl.withDatabase({
required Schema schema,
required SqliteDatabase database,
Logger? logger,
}) {
throw UnimplementedError();
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -297,6 +297,13 @@ mixin PowerSyncDatabaseMixin implements SqliteConnection {
params: params,
);

if (schema.rawTables.isNotEmpty &&
resolvedOptions.source.syncImplementation !=
SyncClientImplementation.rust) {
throw UnsupportedError(
'Raw tables are only supported by the Rust client.');
}

// ignore: deprecated_member_use_from_same_package
clientParams = params;
var thisConnectAborter = AbortController();
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import 'dart:async';
import 'dart:convert';
import 'package:meta/meta.dart';
import 'package:http/browser_client.dart';
import 'package:logging/logging.dart';
Expand Down Expand Up @@ -75,8 +76,12 @@ class PowerSyncDatabaseImpl
SqliteConnectionSetup? sqliteSetup}) {
// ignore: deprecated_member_use_from_same_package
DefaultSqliteOpenFactory factory = PowerSyncOpenFactory(path: path);
return PowerSyncDatabaseImpl.withFactory(factory,
maxReaders: maxReaders, logger: logger, schema: schema);
return PowerSyncDatabaseImpl.withFactory(
factory,
maxReaders: maxReaders,
logger: logger,
schema: schema,
);
}

/// Open a [PowerSyncDatabase] with a [PowerSyncOpenFactory].
Expand All @@ -94,16 +99,22 @@ class PowerSyncDatabaseImpl
Logger? logger}) {
final db = SqliteDatabase.withFactory(openFactory, maxReaders: 1);
return PowerSyncDatabaseImpl.withDatabase(
schema: schema, logger: logger, database: db);
schema: schema,
logger: logger,
database: db,
);
}

/// Open a PowerSyncDatabase on an existing [SqliteDatabase].
///
/// Migrations are run on the database when this constructor is called.
///
/// [logger] defaults to [autoLogger], which logs to the console in debug builds.
PowerSyncDatabaseImpl.withDatabase(
{required this.schema, required this.database, Logger? logger}) {
PowerSyncDatabaseImpl.withDatabase({
required this.schema,
required this.database,
Logger? logger,
}) {
if (logger != null) {
this.logger = logger;
} else {
Expand Down Expand Up @@ -141,6 +152,7 @@ class PowerSyncDatabaseImpl

sync = StreamingSyncImplementation(
adapter: storage,
schemaJson: jsonEncode(schema),
connector: InternalConnector.wrap(connector, this),
crudUpdateTriggerStream: crudStream,
options: options,
Expand Down
130 changes: 128 additions & 2 deletions packages/powersync_core/lib/src/schema.dart
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,23 @@ import 'schema_logic.dart';
/// No migrations are required on the client.
class Schema {
/// List of tables in the schema.
///
/// When opening a PowerSync database, these tables will be created and
/// migrated automatically.
final List<Table> tables;

const Schema(this.tables);
/// A list of [RawTable]s in addition to PowerSync-managed [tables].
///
/// Raw tables give users full control over the SQLite tables, but that
/// includes the responsibility to create those tables and to write migrations
/// for them.
///
/// For more information on raw tables, see [RawTable] and [the documentation](https://docs.powersync.com/usage/use-case-examples/raw-tables).
final List<RawTable> rawTables;

const Schema(this.tables, {this.rawTables = const []});

Map<String, dynamic> toJson() => {'tables': tables};
Map<String, dynamic> toJson() => {'raw_tables': rawTables, 'tables': tables};

void validate() {
Set<String> tableNames = {};
Expand Down Expand Up @@ -315,6 +327,120 @@ class Column {
Map<String, dynamic> toJson() => {'name': name, 'type': type.sqlite};
}

/// A raw table, defined by the user instead of being managed by PowerSync.
///
/// Any ordinary SQLite table can be defined as a raw table, which enables:
///
/// - More performant queries, since data is stored in typed rows instead of the
/// schemaless JSON view PowerSync uses by default.
/// - More control over the table, since custom column constraints can be used
/// in its definition.
///
/// PowerSync doesn't know anything about the internal structure of raw tables -
/// instead, it relies on user-defined [put] and [delete] statements to sync
/// data into them.
///
/// When using raw tables, you are responsible for creating and migrating them
/// when they've changed. Further, triggers are necessary to collect local
/// writes to those tables. For more information, see
/// [the documentation](https://docs.powersync.com/usage/use-case-examples/raw-tables).
///
/// Note that raw tables are only supported by the Rust sync client, which needs
/// to be enabled when connecting with raw tables.
final class RawTable {
/// The name of the table as used by the sync service.
///
/// This doesn't necessarily have to match the name of the SQLite table that
/// [put] and [delete] write to. Instead, it's used by the sync client to
/// identify which statements to use when it encounters sync operations for
/// this table.
final String name;

/// A statement responsible for inserting or updating a row in this raw table
/// based on data from the sync service.
///
/// See [PendingStatement] for details.
final PendingStatement put;

/// A statement responsible for deleting a row based on its PowerSync id.
///
/// See [PendingStatement] for details. Note that [PendingStatementValue]s
/// used here must all be [PendingStatementValue.id].
final PendingStatement delete;

const RawTable({
required this.name,
required this.put,
required this.delete,
});

Map<String, dynamic> toJson() => {
'name': name,
'put': put,
'delete': delete,
};
}

/// An SQL statement to be run by the sync client against raw tables.
///
/// Since raw tables are managed by the user, PowerSync can't know how to apply
/// serverside changes to them. These statements bridge raw tables and PowerSync
/// by providing upserts and delete statements.
///
/// For more information, see [the documentation](https://docs.powersync.com/usage/use-case-examples/raw-tables)
final class PendingStatement {
/// The SQL statement to run to upsert or delete data from a raw table.
final String sql;

/// A list of value identifiers for parameters in [sql].
///
/// Put statements can use both [PendingStatementValue.id] and
/// [PendingStatementValue.column], whereas delete statements can only use
/// [PendingStatementValue.id].
final List<PendingStatementValue> params;

PendingStatement({required this.sql, required this.params});

Map<String, dynamic> toJson() => {
'sql': sql,
'params': params,
};
}

/// A description of a value that will be resolved in the sync client when
/// running a [PendingStatement] for a [RawTable].
sealed class PendingStatementValue {
/// A value that is bound to the textual id used in the PowerSync protocol.
factory PendingStatementValue.id() = _PendingStmtValueId;

/// A value that is bound to the value of a column in a replace (`PUT`)
/// operation of the PowerSync protocol.
factory PendingStatementValue.column(String column) = _PendingStmtValueColumn;

dynamic toJson();
}

class _PendingStmtValueColumn implements PendingStatementValue {
final String column;
const _PendingStmtValueColumn(this.column);

@override
dynamic toJson() {
return {
'Column': column,
};
}
}

class _PendingStmtValueId implements PendingStatementValue {
const _PendingStmtValueId();

@override
dynamic toJson() {
return 'Id';
}
}

/// Type of column.
enum ColumnType {
/// TEXT column.
Expand Down
Loading
Loading