From 11716e3a2f68b0c51f151dd8200ded7450a4aefc Mon Sep 17 00:00:00 2001 From: Martin Kamleithner <martin.kamleithner@gmail.com> Date: Sun, 9 Feb 2025 13:47:04 +0000 Subject: [PATCH 1/2] feat(ferry_sqlite_store): initial implementation --- benchmark/ferry_store_benchmark/.gitignore | 7 + benchmark/ferry_store_benchmark/CHANGELOG.md | 3 + benchmark/ferry_store_benchmark/README.md | 39 ++ .../analysis_options.yaml | 8 + .../ferry_store_benchmark/bin/benchmark.dart | 184 +++++++++ benchmark/ferry_store_benchmark/pubspec.yaml | 29 ++ packages/ferry_sqlite_store/.gitignore | 7 + packages/ferry_sqlite_store/CHANGELOG.md | 3 + packages/ferry_sqlite_store/README.md | 39 ++ .../ferry_sqlite_store/analysis_options.yaml | 30 ++ .../lib/ferry_sqlite_store.dart | 3 + .../lib/src/ferry_sqlite_store_base.dart | 381 ++++++++++++++++++ packages/ferry_sqlite_store/pubspec.yaml | 17 + .../test/ferry_sqlite_store_test.dart | 122 ++++++ 14 files changed, 872 insertions(+) create mode 100644 benchmark/ferry_store_benchmark/.gitignore create mode 100644 benchmark/ferry_store_benchmark/CHANGELOG.md create mode 100644 benchmark/ferry_store_benchmark/README.md create mode 100644 benchmark/ferry_store_benchmark/analysis_options.yaml create mode 100644 benchmark/ferry_store_benchmark/bin/benchmark.dart create mode 100644 benchmark/ferry_store_benchmark/pubspec.yaml create mode 100644 packages/ferry_sqlite_store/.gitignore create mode 100644 packages/ferry_sqlite_store/CHANGELOG.md create mode 100644 packages/ferry_sqlite_store/README.md create mode 100644 packages/ferry_sqlite_store/analysis_options.yaml create mode 100644 packages/ferry_sqlite_store/lib/ferry_sqlite_store.dart create mode 100644 packages/ferry_sqlite_store/lib/src/ferry_sqlite_store_base.dart create mode 100644 packages/ferry_sqlite_store/pubspec.yaml create mode 100644 packages/ferry_sqlite_store/test/ferry_sqlite_store_test.dart diff --git a/benchmark/ferry_store_benchmark/.gitignore b/benchmark/ferry_store_benchmark/.gitignore new file mode 100644 index 00000000..3cceda55 --- /dev/null +++ b/benchmark/ferry_store_benchmark/.gitignore @@ -0,0 +1,7 @@ +# https://dart.dev/guides/libraries/private-files +# Created by `dart pub` +.dart_tool/ + +# Avoid committing pubspec.lock for library packages; see +# https://dart.dev/guides/libraries/private-files#pubspeclock. +pubspec.lock diff --git a/benchmark/ferry_store_benchmark/CHANGELOG.md b/benchmark/ferry_store_benchmark/CHANGELOG.md new file mode 100644 index 00000000..effe43c8 --- /dev/null +++ b/benchmark/ferry_store_benchmark/CHANGELOG.md @@ -0,0 +1,3 @@ +## 1.0.0 + +- Initial version. diff --git a/benchmark/ferry_store_benchmark/README.md b/benchmark/ferry_store_benchmark/README.md new file mode 100644 index 00000000..8831761b --- /dev/null +++ b/benchmark/ferry_store_benchmark/README.md @@ -0,0 +1,39 @@ +<!-- +This README describes the package. If you publish this package to pub.dev, +this README's contents appear on the landing page for your package. + +For information about how to write a good package README, see the guide for +[writing package pages](https://dart.dev/tools/pub/writing-package-pages). + +For general information about developing packages, see the Dart guide for +[creating packages](https://dart.dev/guides/libraries/create-packages) +and the Flutter guide for +[developing packages and plugins](https://flutter.dev/to/develop-packages). +--> + +TODO: Put a short description of the package here that helps potential users +know whether this package might be useful for them. + +## Features + +TODO: List what your package can do. Maybe include images, gifs, or videos. + +## Getting started + +TODO: List prerequisites and provide or point to information on how to +start using the package. + +## Usage + +TODO: Include short and useful examples for package users. Add longer examples +to `/example` folder. + +```dart +const like = 'sample'; +``` + +## Additional information + +TODO: Tell users more about the package: where to find more information, how to +contribute to the package, how to file issues, what response they can expect +from the package authors, and more. diff --git a/benchmark/ferry_store_benchmark/analysis_options.yaml b/benchmark/ferry_store_benchmark/analysis_options.yaml new file mode 100644 index 00000000..d02fc428 --- /dev/null +++ b/benchmark/ferry_store_benchmark/analysis_options.yaml @@ -0,0 +1,8 @@ + +include: package:lints/recommended.yaml + +analyzer: + language: + strict-raw-types: true + strict-casts: true + strict-inference: true diff --git a/benchmark/ferry_store_benchmark/bin/benchmark.dart b/benchmark/ferry_store_benchmark/bin/benchmark.dart new file mode 100644 index 00000000..a117f4d1 --- /dev/null +++ b/benchmark/ferry_store_benchmark/bin/benchmark.dart @@ -0,0 +1,184 @@ +import 'package:benchmark_harness/perf_benchmark_harness.dart'; +import 'package:ferry_hive_ce_store/ferry_hive_ce_store.dart'; +import 'package:ferry_sqlite_store/ferry_sqlite_store.dart'; +import 'package:hive_ce/hive.dart'; +import 'package:sqlite3/sqlite3.dart'; + +class HiveBenchmark extends PerfBenchmarkBase { + final HiveStore store; + + HiveBenchmark(Box<dynamic> box) + : store = HiveStore(box), + super('Hive'); + + @override + Future<void> setup() async { + prepareData(store); + } + + @override + Future<void> run() async { + benchmarkStoreGet(store); + } +} + +class Sqlite3Benchmark extends PerfBenchmarkBase { + final Sqlite3StoreInMemoryFlush store; + + Sqlite3Benchmark(Database db) + : store = Sqlite3StoreInMemoryFlush(db), + super('Sqlite3'); + + @override + void setup() { + prepareData(store); + } + + @override + Future<void> run() async { + benchmarkStoreGet(store); + } +} + +class Sqlite3StoreWriteThrough extends PerfBenchmarkBase { + final Sqlite3StoreInMemoryWriteThrough store; + + Sqlite3StoreWriteThrough(Database db) + : store = Sqlite3StoreInMemoryWriteThrough(db), + super('Sqlite3 Write Through'); + + @override + void setup() { + prepareData(store); + } + + @override + Future<void> run() async { + benchmarkStoreGet(store); + } +} + +class Sqlite3StoreWriteThroughPut extends PerfBenchmarkBase { + final Sqlite3StoreInMemoryWriteThrough store; + + Sqlite3StoreWriteThroughPut(Database db) + : store = Sqlite3StoreInMemoryWriteThrough(db), + super('Sqlite3 Write Through Put'); + + @override + void setup() { + prepareData(store); + } + + @override + Future<void> run() async { + benchMarkStorePut(store); + } +} + +class HiveStorePut extends PerfBenchmarkBase { + final HiveStore store; + + HiveStorePut(Box<dynamic> box) + : store = HiveStore(box), + super('Hive Put'); + + @override + Future<void> setup() async { + prepareData(store); + } + + @override + Future<void> run() async { + benchMarkStorePut(store); + } +} + +class Sqlite3StorePut extends PerfBenchmarkBase { + final Sqlite3StoreInMemoryFlush store; + + Sqlite3StorePut(Database db) + : store = Sqlite3StoreInMemoryFlush(db), + super('Sqlite3 Put'); + + @override + void setup() { + prepareData(store); + } + + @override + Future<void> run() async { + benchMarkStorePut(store); + } +} + +void main() async { + Hive.init('./test/__hive_data__'); + final hiveBox = await Hive.openBox<dynamic>('graphql'); + //WAL + final sqlite3Db = + sqlite3.open('/tmp/ferry_sqlite3.db', mode: OpenMode.readWriteCreate); + + final hiveBenchmark = HiveBenchmark(hiveBox); + final sqlite3Benchmark = Sqlite3Benchmark(sqlite3Db); + + final sqlite3Db2 = + sqlite3.open('/tmp/ferry_sqlite3_2.db', mode: OpenMode.readWriteCreate); + + final sqlite3StoreWriteThrough = Sqlite3StoreWriteThrough(sqlite3Db2); + + hiveBenchmark.report(); + sqlite3Benchmark.report(); + sqlite3StoreWriteThrough.report(); + + final sqlite3Db3 = + sqlite3.open('/tmp/ferry_sqlite3_3.db', mode: OpenMode.readWriteCreate); + + final sqlite3StoreWriteThroughPut = Sqlite3StoreWriteThroughPut(sqlite3Db3); + + sqlite3StoreWriteThroughPut.report(); + + final hiveBox2 = await Hive.openBox<dynamic>('graphql2'); + + final hiveStorePut = HiveStorePut(hiveBox2); + + hiveStorePut.report(); + + final sqlite3Db4 = + sqlite3.open('/tmp/ferry_sqlite3_4.db', mode: OpenMode.readWriteCreate); + + final sqlite3StorePut = Sqlite3StorePut(sqlite3Db4); + + sqlite3StorePut.report(); +} + +void prepareData(Store store) { + final data = { + 'Query': { + 'posts': [ + {'\$ref': 'Post:123'} + ] + }, + 'Post:123': { + 'id': '123', + '__typename': 'Post', + }, + }; + + store.putAll(data); +} + +void benchmarkStoreGet(Store store) { + for (var key in ['Query', 'Post:123']) { + store.get(key); + } +} + +void benchMarkStorePut(Store store) { + for (int i = 0; i < 1000; i++) { + store.put('Post:$i', { + 'id': '$i', + '__typename': 'Post', + }); + } +} diff --git a/benchmark/ferry_store_benchmark/pubspec.yaml b/benchmark/ferry_store_benchmark/pubspec.yaml new file mode 100644 index 00000000..ead9d05f --- /dev/null +++ b/benchmark/ferry_store_benchmark/pubspec.yaml @@ -0,0 +1,29 @@ +name: ferry_store_benchmark +description: A starting point for Dart libraries or applications. +version: 1.0.0 +publish_to: none +# repository: https://github.com/my_org/my_repo + +environment: + sdk: ">=3.0.0 <4.0.0" + +# Add regular dependencies here. +dependencies: + ferry_cache: + path: ../../packages/ferry_cache + ferry_store: + path: ../../packages/ferry_store + ferry_hive_ce_store: + path: ../../packages/ferry_hive_ce_store + ferry_sqlite_store: + path: ../../packages/ferry_sqlite_store + benchmark_harness: ^2.3.1 + hive_ce: ^2.0.0 + sqlite3: ^2.7.3 + rxdart: ^0.28.0 + + + +dev_dependencies: + lints: ^5.0.0 + test: ^1.24.0 diff --git a/packages/ferry_sqlite_store/.gitignore b/packages/ferry_sqlite_store/.gitignore new file mode 100644 index 00000000..3cceda55 --- /dev/null +++ b/packages/ferry_sqlite_store/.gitignore @@ -0,0 +1,7 @@ +# https://dart.dev/guides/libraries/private-files +# Created by `dart pub` +.dart_tool/ + +# Avoid committing pubspec.lock for library packages; see +# https://dart.dev/guides/libraries/private-files#pubspeclock. +pubspec.lock diff --git a/packages/ferry_sqlite_store/CHANGELOG.md b/packages/ferry_sqlite_store/CHANGELOG.md new file mode 100644 index 00000000..effe43c8 --- /dev/null +++ b/packages/ferry_sqlite_store/CHANGELOG.md @@ -0,0 +1,3 @@ +## 1.0.0 + +- Initial version. diff --git a/packages/ferry_sqlite_store/README.md b/packages/ferry_sqlite_store/README.md new file mode 100644 index 00000000..8831761b --- /dev/null +++ b/packages/ferry_sqlite_store/README.md @@ -0,0 +1,39 @@ +<!-- +This README describes the package. If you publish this package to pub.dev, +this README's contents appear on the landing page for your package. + +For information about how to write a good package README, see the guide for +[writing package pages](https://dart.dev/tools/pub/writing-package-pages). + +For general information about developing packages, see the Dart guide for +[creating packages](https://dart.dev/guides/libraries/create-packages) +and the Flutter guide for +[developing packages and plugins](https://flutter.dev/to/develop-packages). +--> + +TODO: Put a short description of the package here that helps potential users +know whether this package might be useful for them. + +## Features + +TODO: List what your package can do. Maybe include images, gifs, or videos. + +## Getting started + +TODO: List prerequisites and provide or point to information on how to +start using the package. + +## Usage + +TODO: Include short and useful examples for package users. Add longer examples +to `/example` folder. + +```dart +const like = 'sample'; +``` + +## Additional information + +TODO: Tell users more about the package: where to find more information, how to +contribute to the package, how to file issues, what response they can expect +from the package authors, and more. diff --git a/packages/ferry_sqlite_store/analysis_options.yaml b/packages/ferry_sqlite_store/analysis_options.yaml new file mode 100644 index 00000000..dee8927a --- /dev/null +++ b/packages/ferry_sqlite_store/analysis_options.yaml @@ -0,0 +1,30 @@ +# This file configures the static analysis results for your project (errors, +# warnings, and lints). +# +# This enables the 'recommended' set of lints from `package:lints`. +# This set helps identify many issues that may lead to problems when running +# or consuming Dart code, and enforces writing Dart using a single, idiomatic +# style and format. +# +# If you want a smaller set of lints you can change this to specify +# 'package:lints/core.yaml'. These are just the most critical lints +# (the recommended set includes the core lints). +# The core lints are also what is used by pub.dev for scoring packages. + +include: package:lints/recommended.yaml + +# Uncomment the following section to specify additional rules. + +# linter: +# rules: +# - camel_case_types + +# analyzer: +# exclude: +# - path/to/excluded/files/** + +# For more information about the core and recommended set of lints, see +# https://dart.dev/go/core-lints + +# For additional information about configuring this file, see +# https://dart.dev/guides/language/analysis-options diff --git a/packages/ferry_sqlite_store/lib/ferry_sqlite_store.dart b/packages/ferry_sqlite_store/lib/ferry_sqlite_store.dart new file mode 100644 index 00000000..18aed4b2 --- /dev/null +++ b/packages/ferry_sqlite_store/lib/ferry_sqlite_store.dart @@ -0,0 +1,3 @@ +library; + +export 'src/ferry_sqlite_store_base.dart'; diff --git a/packages/ferry_sqlite_store/lib/src/ferry_sqlite_store_base.dart b/packages/ferry_sqlite_store/lib/src/ferry_sqlite_store_base.dart new file mode 100644 index 00000000..17a17ee7 --- /dev/null +++ b/packages/ferry_sqlite_store/lib/src/ferry_sqlite_store_base.dart @@ -0,0 +1,381 @@ +import 'dart:async'; +import 'dart:convert'; +import 'dart:typed_data'; + +import 'package:ferry_store/ferry_store.dart'; +import 'package:sqlite3/sqlite3.dart'; +import 'package:rxdart/rxdart.dart'; + +final _jsonUtf8Encoder = const JsonEncoder().fuse(const Utf8Encoder()); +final _jsonUtf8Decoder = const Utf8Decoder().fuse(const JsonDecoder()); + +/// A store that loads data from a SQLite table into memory at startup, +/// and writes back to disk only on [flush] or [dispose]. +/// Uses BehaviorSubject so that watch() immediately yields +/// the current value and doesn't leak memory when unsubscribed. +/// Note: if you're using this in a Flutter app, consider flushing +/// in [WidgetsBindingObserver.didChangeAppLifecycleState]. +/// If you don't flush, you will not persist changes to disk. +class Sqlite3StoreInMemoryFlush extends Store { + final Database _db; + final JsonEquals _jsonEquals; + + // Holds everything in memory to skip JSON parse each time. + final Map<String, Map<String, dynamic>?> _memoryData = {}; + + // Each dataId has its own BehaviorSubject, so new subscribers + // always get the latest value immediately. We'll remove them + // from this map when the last subscriber unsubscribes. + final Map<String, BehaviorSubject<Map<String, dynamic>?>> _watchers = {}; + + // Precompiled statements + late final _deleteStmt = + _db.prepare('DELETE FROM ferry_store WHERE data_id = ?'); + late final _putStmt = _db.prepare( + 'INSERT OR REPLACE INTO ferry_store (data_id, data) VALUES (?, ?)', + ); + + Sqlite3StoreInMemoryFlush(this._db, [JsonEquals? jsonEquals]) + : _jsonEquals = jsonEquals ?? jsonMapEquals { + _ensureInitialized(); + _loadAllFromDb(); + } + + void _ensureInitialized() { + _db.execute('PRAGMA journal_mode = WAL;'); + _db.execute('CREATE TABLE IF NOT EXISTS ferry_store (' + 'data_id TEXT PRIMARY KEY NOT NULL,' + 'data BLOB' + ')'); + _db.execute( + 'CREATE UNIQUE INDEX IF NOT EXISTS ferry_store_data_id ' + 'ON ferry_store (data_id)', + ); + } + + void _loadAllFromDb() { + final result = _db.select('SELECT data_id, data FROM ferry_store'); + for (final row in result) { + final dataId = row['data_id'] as String; + final bytes = row['data'] as Uint8List; + final json = _jsonUtf8Decoder.convert(bytes); + _memoryData[dataId] = json as Map<String, dynamic>?; + } + } + + @override + void clear() { + _memoryData.clear(); + // No DB write until flush + _notifyWatchersAll(null); + } + + @override + void delete(String dataId) { + if (!_memoryData.containsKey(dataId)) return; + _memoryData.remove(dataId); + // No DB write until flush + _notifyWatcher(dataId, null); + } + + @override + void deleteAll(Iterable<String> dataIds) { + for (final id in dataIds) { + if (_memoryData.containsKey(id)) { + _memoryData.remove(id); + _notifyWatcher(id, null); + } + } + } + + @override + + /// Get the data from memory. + /// You MUST NOT modify the returned map. + Map<String, dynamic>? get(String dataId) { + return _memoryData[dataId]; + } + + @override + Iterable<String> get keys => _memoryData.keys; + + @override + void put(String dataId, Map<String, dynamic>? value) { + final oldValue = _memoryData[dataId]; + _memoryData[dataId] = value; + if (!_jsonEquals(oldValue, value)) { + _notifyWatcher(dataId, value); + } + } + + @override + void putAll(Map<String, Map<String, dynamic>?> data) { + data.forEach((dataId, newValue) { + final oldValue = _memoryData[dataId]; + _memoryData[dataId] = newValue; + if (!_jsonEquals(oldValue, newValue)) { + _notifyWatcher(dataId, newValue); + } + }); + } + + /// Writes all in-memory data to the database. + @override + Future<void> flush() async { + try { + _db.execute('BEGIN TRANSACTION;'); + _db.execute('DELETE FROM ferry_store;'); + + for (final entry in _memoryData.entries) { + final dataId = entry.key; + final value = entry.value; + final blob = _jsonUtf8Encoder.convert(value); + _putStmt.execute([dataId, blob]); + } + _db.execute('COMMIT;'); + } catch (e) { + _db.execute('ROLLBACK;'); + rethrow; + } + } + + @override + Future<void> dispose() async { + // Make sure we persist changes before closing + await flush(); + _deleteStmt.dispose(); + _putStmt.dispose(); + _db.dispose(); + + // Close watchers + for (final subject in _watchers.values) { + await subject.close(); + } + _watchers.clear(); + } + + /// Return a stream that immediately emits the current value. + /// Because we use BehaviorSubject, new subscribers will always see + /// the "most recent" event as soon as they listen. + /// + /// We also apply `.distinct()` to avoid spamming watchers if `distinct = true`. + @override + Stream<Map<String, dynamic>?> watch(String dataId, {bool distinct = true}) { + // If there's already a BehaviorSubject for this dataId, use it. + final subject = _watchers.putIfAbsent(dataId, () { + // Create a new subject seeded with the current in-memory value. + final s = + BehaviorSubject<Map<String, dynamic>?>.seeded(_memoryData[dataId]); + + // When the last subscriber cancels, automatically close + remove from the map + s.onCancel = () { + // If no more listeners remain on this subject, close and remove it + if (!s.hasListener) { + s.close(); + _watchers.remove(dataId); + } + }; + return s; + }); + + // If distinct, filter out consecutive duplicates: + if (distinct) { + return subject.distinct((prev, next) => _jsonEquals(prev, next)); + } else { + return subject; + } + } + + /// Notify watchers of a specific dataId about new value. + void _notifyWatcher(String dataId, Map<String, dynamic>? newValue) { + _watchers[dataId]?.add(newValue); + } + + /// Notify all watchers that everything changed (e.g. cleared). + void _notifyWatchersAll(Map<String, dynamic>? newValue) { + for (final subject in _watchers.values) { + subject.add(newValue); + } + } +} + +/// A store that reads/writes from/to a SQLite table, +/// but also keeps an in-memory map to avoid repeated JSON parsing on [get]. +/// On every [put]/[delete], it writes to the database immediately. +/// It uses BehaviorSubject so that watch() immediately yields +/// the current value and cleans up watchers when they're unsubscribed. +class Sqlite3StoreInMemoryWriteThrough extends Store { + final Database _db; + final JsonEquals _jsonEquals; + + // Holds everything in memory to skip JSON parse each time. + final Map<String, Map<String, dynamic>?> _memoryData = {}; + + // Each dataId gets its own BehaviorSubject with the latest value. + // We'll remove the subject once all subscribers unsubscribe. + final Map<String, BehaviorSubject<Map<String, dynamic>?>> _watchers = {}; + + // Precompiled statements + late final _deleteStmt = + _db.prepare('DELETE FROM ferry_store WHERE data_id = ?'); + + late final _putStmt = _db.prepare( + 'INSERT OR REPLACE INTO ferry_store (data_id, data) VALUES (?, ?)'); + + Sqlite3StoreInMemoryWriteThrough(this._db, [JsonEquals? jsonEquals]) + : _jsonEquals = jsonEquals ?? jsonMapEquals { + _ensureInitialized(); + _loadAllFromDb(); + } + + void _ensureInitialized() { + _db.execute('PRAGMA journal_mode = WAL;' + 'CREATE TABLE IF NOT EXISTS ferry_store (data_id TEXT PRIMARY KEY NOT NULL, data BLOB); ' + 'CREATE UNIQUE INDEX IF NOT EXISTS ferry_store_data_id ON ferry_store (data_id);'); + } + + void _loadAllFromDb() { + final result = _db.select('SELECT data_id, data FROM ferry_store'); + for (final row in result) { + final dataId = row['data_id'] as String; + final bytes = row['data'] as Uint8List; + final json = _jsonUtf8Decoder.convert(bytes); + _memoryData[dataId] = json as Map<String, dynamic>?; + } + } + + @override + void clear() { + _memoryData.clear(); + // Immediately write to DB + _db.execute('DELETE FROM ferry_store;'); + // Notify watchers + _notifyWatchersAll(null); + } + + @override + void delete(String dataId) { + if (!_memoryData.containsKey(dataId)) { + return; + } + _memoryData.remove(dataId); + // Immediately write to DB + _deleteStmt.execute([dataId]); + // Notify watchers + _notifyWatcher(dataId, null); + } + + @override + void deleteAll(Iterable<String> dataIds) { + for (final id in dataIds) { + if (_memoryData.containsKey(id)) { + _memoryData.remove(id); + _deleteStmt.execute([id]); + _notifyWatcher(id, null); + } + } + } + + @override + Map<String, dynamic>? get(String dataId) { + return _memoryData[dataId]; + } + + @override + Iterable<String> get keys => _memoryData.keys; + + @override + void put(String dataId, Map<String, dynamic>? value) { + final oldValue = _memoryData[dataId]; + // If the new value is the same, skip the DB write & watchers + if (_jsonEquals(oldValue, value)) { + return; + } + + // Write to memory + _memoryData[dataId] = value; + + // Immediately write to DB + final blob = _jsonUtf8Encoder.convert(value); + _putStmt.execute([dataId, blob]); + + // Notify watchers + _notifyWatcher(dataId, value); + } + + @override + void putAll(Map<String, Map<String, dynamic>?> data) { + data.forEach((dataId, newValue) { + final oldValue = _memoryData[dataId]; + if (_jsonEquals(oldValue, newValue)) { + return; + } + + _memoryData[dataId] = newValue; + final blob = _jsonUtf8Encoder.convert(newValue); + _putStmt.execute([dataId, blob]); + + _notifyWatcher(dataId, newValue); + }); + } + + /// In write-through, everything is already saved to DB as we go. + @override + Future<void> flush() async { + // No-op: data is already persisted on each put/delete + } + + @override + Future<void> dispose() async { + // Optional: flush in case you have leftover changes in memory + await flush(); + + _deleteStmt.dispose(); + _putStmt.dispose(); + _db.dispose(); + + // Close watchers + for (final subject in _watchers.values) { + await subject.close(); + } + _watchers.clear(); + } + + /// Return a stream that immediately emits the current value (BehaviorSubject) + /// and then any future updates. Each dataId has its own subject. + /// We remove the subject once all subscribers unsubscribe. + @override + Stream<Map<String, dynamic>?> watch(String dataId, {bool distinct = true}) { + final subject = _watchers.putIfAbsent(dataId, () { + // Seed the subject with the current in-memory value + final s = + BehaviorSubject<Map<String, dynamic>?>.seeded(_memoryData[dataId]); + + // Auto-cleanup when the last subscriber unsubscribes + s.onCancel = () { + if (!s.hasListener) { + s.close(); + _watchers.remove(dataId); + } + }; + + return s; + }); + + if (distinct) { + return subject.distinct((prev, next) => _jsonEquals(prev, next)); + } else { + return subject; + } + } + + void _notifyWatcher(String dataId, Map<String, dynamic>? newValue) { + _watchers[dataId]?.add(newValue); + } + + void _notifyWatchersAll(Map<String, dynamic>? newValue) { + for (final subject in _watchers.values) { + subject.add(newValue); + } + } +} diff --git a/packages/ferry_sqlite_store/pubspec.yaml b/packages/ferry_sqlite_store/pubspec.yaml new file mode 100644 index 00000000..1897f290 --- /dev/null +++ b/packages/ferry_sqlite_store/pubspec.yaml @@ -0,0 +1,17 @@ +name: ferry_sqlite_store +description: A starting point for Dart libraries or applications. +version: 1.0.0 +# repository: https://github.com/my_org/my_repo + +environment: + sdk: '>=3.0.0 <4.0.0' + +# Add regular dependencies here. +dependencies: + ferry_store: ^0.6.1 + sqlite3: ^2.7.3 + rxdart: ^0.28.0 + +dev_dependencies: + lints: ^5.0.0 + test: ^1.24.0 diff --git a/packages/ferry_sqlite_store/test/ferry_sqlite_store_test.dart b/packages/ferry_sqlite_store/test/ferry_sqlite_store_test.dart new file mode 100644 index 00000000..e1d4888b --- /dev/null +++ b/packages/ferry_sqlite_store/test/ferry_sqlite_store_test.dart @@ -0,0 +1,122 @@ +import 'package:ferry_sqlite_store/ferry_sqlite_store.dart'; +import 'package:sqlite3/sqlite3.dart'; +import 'package:test/test.dart'; + +void main() { + group('A group of tests', () { + late Sqlite3StoreInMemoryFlush store; + + setUp(() { + store = Sqlite3StoreInMemoryFlush(sqlite3.openInMemory()); + }); + + tearDown(() { + return store.dispose(); + }); + + test('can get data', () async { + final data = { + 'Query': { + 'posts': [ + {'\$ref': 'Post:123'} + ] + }, + 'Post:123': { + 'id': '123', + '__typename': 'Post', + }, + }; + + store.putAll(data); + + for (var entry in data.entries) { + expect(store.get(entry.key), equals(entry.value)); + } + }); + + test('can put data', () async { + final data = { + 'Query': { + 'posts': [ + {'\$ref': 'Post:123'} + ] + }, + 'Post:123': { + 'id': '123', + '__typename': 'Post', + }, + }; + + for (var entry in data.entries) { + store.put(entry.key, entry.value); + } + + for (var entry in data.entries) { + expect(store.get(entry.key), equals(entry.value)); + } + }); + + test('can put all data', () async { + final data = { + 'Query': { + 'posts': [ + {'\$ref': 'Post:123'} + ] + }, + 'Post:123': { + 'id': '123', + '__typename': 'Post', + }, + }; + + store.putAll(data); + + for (var entry in data.entries) { + expect(store.get(entry.key), equals(entry.value)); + } + }); + + test('can delete data', () async { + final data = { + 'Query': { + 'posts': [ + {'\$ref': 'Post:123'} + ] + }, + 'Post:123': { + 'id': '123', + '__typename': 'Post', + }, + }; + + store.putAll(data); + + final key = store.keys.first; + + store.delete(key); + expect( + store.get(key), + equals(null), + ); + }); + + test('can clear data', () async { + final data = { + 'Query': { + 'posts': [ + {'\$ref': 'Post:123'} + ] + }, + 'Post:123': { + 'id': '123', + '__typename': 'Post', + }, + }; + + store.putAll(data); + + store.clear(); + expect(store.keys.length, equals(0)); + }); + }); +} From f62fe50c843a32e5a82082492522f41a9179fa46 Mon Sep 17 00:00:00 2001 From: Martin Kamleithner <martin.kamleithner@gmail.com> Date: Mon, 10 Feb 2025 15:30:44 +0000 Subject: [PATCH 2/2] feat(ferry_sqlite_store): optimize distinct --- benchmark/ferry_store_benchmark/pubspec.lock | 527 ++++++++++++++++++ .../client/lib/main.dart | 28 +- .../lib/src/ferry_sqlite_store_base.dart | 4 +- 3 files changed, 556 insertions(+), 3 deletions(-) create mode 100644 benchmark/ferry_store_benchmark/pubspec.lock diff --git a/benchmark/ferry_store_benchmark/pubspec.lock b/benchmark/ferry_store_benchmark/pubspec.lock new file mode 100644 index 00000000..d15db385 --- /dev/null +++ b/benchmark/ferry_store_benchmark/pubspec.lock @@ -0,0 +1,527 @@ +# Generated by pub +# See https://dart.dev/tools/pub/glossary#lockfile +packages: + _fe_analyzer_shared: + dependency: transitive + description: + name: _fe_analyzer_shared + sha256: dc27559385e905ad30838356c5f5d574014ba39872d732111cd07ac0beff4c57 + url: "https://pub.dev" + source: hosted + version: "80.0.0" + analyzer: + dependency: transitive + description: + name: analyzer + sha256: "192d1c5b944e7e53b24b5586db760db934b177d4147c42fbca8c8c5f1eb8d11e" + url: "https://pub.dev" + source: hosted + version: "7.3.0" + args: + dependency: transitive + description: + name: args + sha256: bf9f5caeea8d8fe6721a9c358dd8a5c1947b27f1cfaa18b39c301273594919e6 + url: "https://pub.dev" + source: hosted + version: "2.6.0" + async: + dependency: transitive + description: + name: async + sha256: "758e6d74e971c3e5aceb4110bfd6698efc7f501675bcfe0c775459a8140750eb" + url: "https://pub.dev" + source: hosted + version: "2.13.0" + benchmark_harness: + dependency: "direct main" + description: + name: benchmark_harness + sha256: "83f65107165883ba8623eb822daacb23dcf9f795c66841de758c9dd7c5a0cf28" + url: "https://pub.dev" + source: hosted + version: "2.3.1" + boolean_selector: + dependency: transitive + description: + name: boolean_selector + sha256: "8aab1771e1243a5063b8b0ff68042d67334e3feab9e95b9490f9a6ebf73b42ea" + url: "https://pub.dev" + source: hosted + version: "2.1.2" + built_collection: + dependency: transitive + description: + name: built_collection + sha256: "376e3dd27b51ea877c28d525560790aee2e6fbb5f20e2f85d5081027d94e2100" + url: "https://pub.dev" + source: hosted + version: "5.1.1" + built_value: + dependency: transitive + description: + name: built_value + sha256: "28a712df2576b63c6c005c465989a348604960c0958d28be5303ba9baa841ac2" + url: "https://pub.dev" + source: hosted + version: "8.9.3" + collection: + dependency: transitive + description: + name: collection + sha256: "2f5709ae4d3d59dd8f7cd309b4e023046b57d8a6c82130785d2b0e5868084e76" + url: "https://pub.dev" + source: hosted + version: "1.19.1" + convert: + dependency: transitive + description: + name: convert + sha256: b30acd5944035672bc15c6b7a8b47d773e41e2f17de064350988c5d02adb1c68 + url: "https://pub.dev" + source: hosted + version: "3.1.2" + coverage: + dependency: transitive + description: + name: coverage + sha256: e3493833ea012784c740e341952298f1cc77f1f01b1bbc3eb4eecf6984fb7f43 + url: "https://pub.dev" + source: hosted + version: "1.11.1" + crypto: + dependency: transitive + description: + name: crypto + sha256: "1e445881f28f22d6140f181e07737b22f1e099a5e1ff94b0af2f9e4a463f4855" + url: "https://pub.dev" + source: hosted + version: "3.0.6" + ferry_cache: + dependency: "direct main" + description: + path: "../ferry_cache" + relative: true + source: path + version: "0.10.1" + ferry_exec: + dependency: "direct overridden" + description: + path: "../ferry_exec" + relative: true + source: path + version: "0.7.0" + ferry_hive_ce_store: + dependency: "direct main" + description: + path: "../ferry_hive_ce_store" + relative: true + source: path + version: "0.0.1" + ferry_sqlite_store: + dependency: "direct main" + description: + path: "../ferry_sqlite_store" + relative: true + source: path + version: "1.0.0" + ferry_store: + dependency: "direct main" + description: + path: "../ferry_store" + relative: true + source: path + version: "0.6.1" + ffi: + dependency: transitive + description: + name: ffi + sha256: "16ed7b077ef01ad6170a3d0c57caa4a112a38d7a2ed5602e0aca9ca6f3d98da6" + url: "https://pub.dev" + source: hosted + version: "2.1.3" + file: + dependency: transitive + description: + name: file + sha256: a3b4f84adafef897088c160faf7dfffb7696046cb13ae90b508c2cbc95d3b8d4 + url: "https://pub.dev" + source: hosted + version: "7.0.1" + fixnum: + dependency: transitive + description: + name: fixnum + sha256: b6dc7065e46c974bc7c5f143080a6764ec7a4be6da1285ececdc37be96de53be + url: "https://pub.dev" + source: hosted + version: "1.1.1" + frontend_server_client: + dependency: transitive + description: + name: frontend_server_client + sha256: f64a0333a82f30b0cca061bc3d143813a486dc086b574bfb233b7c1372427694 + url: "https://pub.dev" + source: hosted + version: "4.0.0" + glob: + dependency: transitive + description: + name: glob + sha256: c3f1ee72c96f8f78935e18aa8cecced9ab132419e8625dc187e1c2408efc20de + url: "https://pub.dev" + source: hosted + version: "2.1.3" + gql: + dependency: transitive + description: + name: gql + sha256: "994e0c530b5ca9fce716945031b90c7925eb00bd37624c43ad4cb51883f383b3" + url: "https://pub.dev" + source: hosted + version: "1.0.0+1" + gql_exec: + dependency: transitive + description: + name: gql_exec + sha256: "957db95ba37028001f76c1904e18b848f57cfef878e572ce77c6d898fafecf8f" + url: "https://pub.dev" + source: hosted + version: "1.0.0+1" + gql_link: + dependency: transitive + description: + name: gql_link + sha256: "4aa288a1e67c899b0d746a2209e949fd0dc87e56d9b759a5cdc46a7069722cf3" + url: "https://pub.dev" + source: hosted + version: "1.0.0+1" + hive_ce: + dependency: "direct main" + description: + name: hive_ce + sha256: ac66daee46ad46486a1ed12cf91e9d7479c875fb46889be8d2c96b557406647f + url: "https://pub.dev" + source: hosted + version: "2.10.1" + http_multi_server: + dependency: transitive + description: + name: http_multi_server + sha256: aa6199f908078bb1c5efb8d8638d4ae191aac11b311132c3ef48ce352fb52ef8 + url: "https://pub.dev" + source: hosted + version: "3.2.2" + http_parser: + dependency: transitive + description: + name: http_parser + sha256: "178d74305e7866013777bab2c3d8726205dc5a4dd935297175b19a23a2e66571" + url: "https://pub.dev" + source: hosted + version: "4.1.2" + io: + dependency: transitive + description: + name: io + sha256: dfd5a80599cf0165756e3181807ed3e77daf6dd4137caaad72d0b7931597650b + url: "https://pub.dev" + source: hosted + version: "1.0.5" + js: + dependency: transitive + description: + name: js + sha256: c1b2e9b5ea78c45e1a0788d29606ba27dc5f71f019f32ca5140f61ef071838cf + url: "https://pub.dev" + source: hosted + version: "0.7.1" + lints: + dependency: "direct dev" + description: + name: lints + sha256: "4a16b3f03741e1252fda5de3ce712666d010ba2122f8e912c94f9f7b90e1a4c3" + url: "https://pub.dev" + source: hosted + version: "5.1.0" + logging: + dependency: transitive + description: + name: logging + sha256: c8245ada5f1717ed44271ed1c26b8ce85ca3228fd2ffdb75468ab01979309d61 + url: "https://pub.dev" + source: hosted + version: "1.3.0" + matcher: + dependency: transitive + description: + name: matcher + sha256: dc58c723c3c24bf8d3e2d3ad3f2f9d7bd9cf43ec6feaa64181775e60190153f2 + url: "https://pub.dev" + source: hosted + version: "0.12.17" + meta: + dependency: transitive + description: + name: meta + sha256: e3641ec5d63ebf0d9b41bd43201a66e3fc79a65db5f61fc181f04cd27aab950c + url: "https://pub.dev" + source: hosted + version: "1.16.0" + mime: + dependency: transitive + description: + name: mime + sha256: "41a20518f0cb1256669420fdba0cd90d21561e560ac240f26ef8322e45bb7ed6" + url: "https://pub.dev" + source: hosted + version: "2.0.0" + node_preamble: + dependency: transitive + description: + name: node_preamble + sha256: "6e7eac89047ab8a8d26cf16127b5ed26de65209847630400f9aefd7cd5c730db" + url: "https://pub.dev" + source: hosted + version: "2.0.2" + normalize: + dependency: "direct overridden" + description: + path: "../normalize" + relative: true + source: path + version: "0.10.0" + package_config: + dependency: transitive + description: + name: package_config + sha256: "92d4488434b520a62570293fbd33bb556c7d49230791c1b4bbd973baf6d2dc67" + url: "https://pub.dev" + source: hosted + version: "2.1.1" + path: + dependency: transitive + description: + name: path + sha256: "75cca69d1490965be98c73ceaea117e8a04dd21217b37b292c9ddbec0d955bc5" + url: "https://pub.dev" + source: hosted + version: "1.9.1" + pedantic: + dependency: transitive + description: + name: pedantic + sha256: "67fc27ed9639506c856c840ccce7594d0bdcd91bc8d53d6e52359449a1d50602" + url: "https://pub.dev" + source: hosted + version: "1.11.1" + pool: + dependency: transitive + description: + name: pool + sha256: "20fe868b6314b322ea036ba325e6fc0711a22948856475e2c2b6306e8ab39c2a" + url: "https://pub.dev" + source: hosted + version: "1.5.1" + pub_semver: + dependency: transitive + description: + name: pub_semver + sha256: "7b3cfbf654f3edd0c6298ecd5be782ce997ddf0e00531b9464b55245185bbbbd" + url: "https://pub.dev" + source: hosted + version: "2.1.5" + rxdart: + dependency: "direct main" + description: + name: rxdart + sha256: "5c3004a4a8dbb94bd4bf5412a4def4acdaa12e12f269737a5751369e12d1a962" + url: "https://pub.dev" + source: hosted + version: "0.28.0" + shelf: + dependency: transitive + description: + name: shelf + sha256: e7dd780a7ffb623c57850b33f43309312fc863fb6aa3d276a754bb299839ef12 + url: "https://pub.dev" + source: hosted + version: "1.4.2" + shelf_packages_handler: + dependency: transitive + description: + name: shelf_packages_handler + sha256: "89f967eca29607c933ba9571d838be31d67f53f6e4ee15147d5dc2934fee1b1e" + url: "https://pub.dev" + source: hosted + version: "3.0.2" + shelf_static: + dependency: transitive + description: + name: shelf_static + sha256: c87c3875f91262785dade62d135760c2c69cb217ac759485334c5857ad89f6e3 + url: "https://pub.dev" + source: hosted + version: "1.1.3" + shelf_web_socket: + dependency: transitive + description: + name: shelf_web_socket + sha256: "3632775c8e90d6c9712f883e633716432a27758216dfb61bd86a8321c0580925" + url: "https://pub.dev" + source: hosted + version: "3.0.0" + source_map_stack_trace: + dependency: transitive + description: + name: source_map_stack_trace + sha256: c0713a43e323c3302c2abe2a1cc89aa057a387101ebd280371d6a6c9fa68516b + url: "https://pub.dev" + source: hosted + version: "2.1.2" + source_maps: + dependency: transitive + description: + name: source_maps + sha256: "190222579a448b03896e0ca6eca5998fa810fda630c1d65e2f78b3f638f54812" + url: "https://pub.dev" + source: hosted + version: "0.10.13" + source_span: + dependency: transitive + description: + name: source_span + sha256: "254ee5351d6cb365c859e20ee823c3bb479bf4a293c22d17a9f1bf144ce86f7c" + url: "https://pub.dev" + source: hosted + version: "1.10.1" + sqlite3: + dependency: "direct main" + description: + name: sqlite3 + sha256: decd58236d7c59e01ae81b34ebd158e6a1b61e0ae5397fc428736eb91ab82808 + url: "https://pub.dev" + source: hosted + version: "2.7.3" + stack_trace: + dependency: transitive + description: + name: stack_trace + sha256: "8b27215b45d22309b5cddda1aa2b19bdfec9df0e765f2de506401c071d38d1b1" + url: "https://pub.dev" + source: hosted + version: "1.12.1" + stream_channel: + dependency: transitive + description: + name: stream_channel + sha256: "969e04c80b8bcdf826f8f16579c7b14d780458bd97f56d107d3950fdbeef059d" + url: "https://pub.dev" + source: hosted + version: "2.1.4" + string_scanner: + dependency: transitive + description: + name: string_scanner + sha256: "921cd31725b72fe181906c6a94d987c78e3b98c2e205b397ea399d4054872b43" + url: "https://pub.dev" + source: hosted + version: "1.4.1" + term_glyph: + dependency: transitive + description: + name: term_glyph + sha256: "7f554798625ea768a7518313e58f83891c7f5024f88e46e7182a4558850a4b8e" + url: "https://pub.dev" + source: hosted + version: "1.2.2" + test: + dependency: "direct dev" + description: + name: test + sha256: "301b213cd241ca982e9ba50266bd3f5bd1ea33f1455554c5abb85d1be0e2d87e" + url: "https://pub.dev" + source: hosted + version: "1.25.15" + test_api: + dependency: transitive + description: + name: test_api + sha256: fb31f383e2ee25fbbfe06b40fe21e1e458d14080e3c67e7ba0acfde4df4e0bbd + url: "https://pub.dev" + source: hosted + version: "0.7.4" + test_core: + dependency: transitive + description: + name: test_core + sha256: "84d17c3486c8dfdbe5e12a50c8ae176d15e2a771b96909a9442b40173649ccaa" + url: "https://pub.dev" + source: hosted + version: "0.6.8" + typed_data: + dependency: transitive + description: + name: typed_data + sha256: f9049c039ebfeb4cf7a7104a675823cd72dba8297f264b6637062516699fa006 + url: "https://pub.dev" + source: hosted + version: "1.4.0" + vm_service: + dependency: transitive + description: + name: vm_service + sha256: ddfa8d30d89985b96407efce8acbdd124701f96741f2d981ca860662f1c0dc02 + url: "https://pub.dev" + source: hosted + version: "15.0.0" + watcher: + dependency: transitive + description: + name: watcher + sha256: "69da27e49efa56a15f8afe8f4438c4ec02eff0a117df1b22ea4aad194fe1c104" + url: "https://pub.dev" + source: hosted + version: "1.1.1" + web: + dependency: transitive + description: + name: web + sha256: cd3543bd5798f6ad290ea73d210f423502e71900302dde696f8bff84bf89a1cb + url: "https://pub.dev" + source: hosted + version: "1.1.0" + web_socket: + dependency: transitive + description: + name: web_socket + sha256: "3c12d96c0c9a4eec095246debcea7b86c0324f22df69893d538fcc6f1b8cce83" + url: "https://pub.dev" + source: hosted + version: "0.1.6" + web_socket_channel: + dependency: transitive + description: + name: web_socket_channel + sha256: "0b8e2457400d8a859b7b2030786835a28a8e80836ef64402abef392ff4f1d0e5" + url: "https://pub.dev" + source: hosted + version: "3.0.2" + webkit_inspection_protocol: + dependency: transitive + description: + name: webkit_inspection_protocol + sha256: "87d3f2333bb240704cd3f1c6b5b7acd8a10e7f0bc28c28dcf14e782014f4a572" + url: "https://pub.dev" + source: hosted + version: "1.2.1" + yaml: + dependency: transitive + description: + name: yaml + sha256: b9da305ac7c39faa3f030eccd175340f968459dae4af175130b3fc47e40d76ce + url: "https://pub.dev" + source: hosted + version: "3.1.3" +sdks: + dart: ">=3.6.0-269.0.dev <4.0.0" diff --git a/examples/auth_token_with_isolate/client/lib/main.dart b/examples/auth_token_with_isolate/client/lib/main.dart index 1378de76..e02f2e21 100644 --- a/examples/auth_token_with_isolate/client/lib/main.dart +++ b/examples/auth_token_with_isolate/client/lib/main.dart @@ -30,9 +30,14 @@ String _getEndpoint() { defaultValue: 'http://localhost:3010/graphql'); } -class MyApp extends StatelessWidget { +class MyApp extends StatefulWidget { const MyApp({super.key}); + @override + State<MyApp> createState() => _MyAppState(); +} + +class _MyAppState extends State<MyApp> with WidgetsBindingObserver { // This widget is the root of your application. @override Widget build(BuildContext context) { @@ -53,6 +58,27 @@ class MyApp extends StatelessWidget { home: const MyHomePage(), ); } + + @override + void initState() { + super.initState(); + WidgetsBinding.instance.addObserver(this); + } + + @override + void dispose() { + WidgetsBinding.instance.removeObserver(this); + super.dispose(); + } + + @override + void didChangeAppLifecycleState(AppLifecycleState state) { + if (state == AppLifecycleState.hidden) { + final ref = ProviderScope.containerOf(context, listen: false); + final client = ref.read(clientProvider); + client.flushStore(); + } + } } class MyHomePage extends StatelessWidget { diff --git a/packages/ferry_sqlite_store/lib/src/ferry_sqlite_store_base.dart b/packages/ferry_sqlite_store/lib/src/ferry_sqlite_store_base.dart index 17a17ee7..cba3e496 100644 --- a/packages/ferry_sqlite_store/lib/src/ferry_sqlite_store_base.dart +++ b/packages/ferry_sqlite_store/lib/src/ferry_sqlite_store_base.dart @@ -180,7 +180,7 @@ class Sqlite3StoreInMemoryFlush extends Store { // If distinct, filter out consecutive duplicates: if (distinct) { - return subject.distinct((prev, next) => _jsonEquals(prev, next)); + return subject.distinct(_jsonEquals); } else { return subject; } @@ -363,7 +363,7 @@ class Sqlite3StoreInMemoryWriteThrough extends Store { }); if (distinct) { - return subject.distinct((prev, next) => _jsonEquals(prev, next)); + return subject.distinct(_jsonEquals); } else { return subject; }