diff --git a/mobile/lib/db/collections_db.dart b/mobile/lib/db/collections_db.dart index 6c57410962..70e9cc3d38 100644 --- a/mobile/lib/db/collections_db.dart +++ b/mobile/lib/db/collections_db.dart @@ -10,6 +10,7 @@ import "package:photos/models/collection/collection_old.dart"; import 'package:sqflite/sqflite.dart'; import 'package:sqflite_migration/sqflite_migration.dart'; +@Deprecated("Use remoteDB instead") class CollectionsDB { static const _databaseName = "ente.collections.db"; static const table = 'collections'; diff --git a/mobile/lib/db/remote/db.dart b/mobile/lib/db/remote/db.dart index 5f2f9cd26d..a8573e92cd 100644 --- a/mobile/lib/db/remote/db.dart +++ b/mobile/lib/db/remote/db.dart @@ -1,32 +1,95 @@ import "dart:developer"; import "dart:io"; +import "package:collection/collection.dart"; +import "package:flutter/foundation.dart"; import "package:path/path.dart"; import "package:path_provider/path_provider.dart"; import "package:photos/db/remote/migration.dart"; +import "package:photos/models/collection/collection.dart"; import "package:sqlite_async/sqlite_async.dart"; var devLog = log; +// ignore: constant_identifier_names +enum RemoteTable { collections, collection_files, files, entities } + class RemoteDB { static const _databaseName = "remote.db"; - // only have a single app-wide reference to the database - static Future? _sqliteAsyncDBFuture; + static const _batchInsertMaxCount = 1000; + late final SqliteDatabase _sqliteDB; - Future get sqliteAsyncDB async { - // lazily instantiate the db the first time it is accessed - _sqliteAsyncDBFuture ??= _initSqliteAsyncDatabase(); - return _sqliteAsyncDBFuture!; - } - - // this opens the database (and creates it if it doesn't exist) - Future _initSqliteAsyncDatabase() async { + Future init() async { + devLog("Starting RemoteDB init"); final Directory documentsDirectory = await getApplicationDocumentsDirectory(); final String path = join(documentsDirectory.path, _databaseName); - devLog("DB path " + path); + final database = SqliteDatabase(path: path); await RemoteDBMigration.migrate(database); - return database; + _sqliteDB = database; + devLog("RemoteDB init complete $path"); + } + + Future> getAllCollections() async { + final result = []; + final cursor = await _sqliteDB.getAll("SELECT * FROM collections"); + for (final row in cursor) { + result.add(Collection.fromRow(row)); + } + return result; + } + + Future insertCollections(List collections) async { + if (collections.isEmpty) return; + final stopwatch = Stopwatch()..start(); + int inserted = 0; + await Future.forEach(collections.slices(_batchInsertMaxCount), + (slice) async { + debugPrint( + '$runtimeType insertCollections inserting slice of [${inserted + 1}, ${inserted + slice.length}] entries', + ); + final List> values = + slice.map((e) => e.rowValiues()).toList(); + await _sqliteDB.executeBatch( + 'INSERT OR REPLACE INTO collections ($collectionColumns) values($collectionValuePlaceHolder)', + values, + ); + inserted += slice.length; + }); + debugPrint( + '$runtimeType insertCollections complete in ${stopwatch.elapsed.inMilliseconds}ms for ${collections.length} collections', + ); + } + + Future deleteEntries(Set ids, RemoteTable table) async { + if (ids.isEmpty) return; + final stopwatch = Stopwatch()..start(); + await _sqliteDB.execute( + 'DELETE FROM ${table.name.toLowerCase()} WHERE id IN (${ids.join(',')})', + ); + debugPrint( + '$runtimeType deleteEntries complete in ${stopwatch.elapsed.inMilliseconds}ms for ${ids.length} $table entries', + ); + } + + Future> _getByIds( + Set ids, + String table, + T Function( + Map row, + ) mapRow, { + String columnName = "id", + }) async { + final result = {}; + if (ids.isNotEmpty) { + final rows = await _sqliteDB.getAll( + 'SELECT * from $table where $columnName IN (${ids.join(',')})', + ); + for (final row in rows) { + result.add(mapRow(row)); + } + } + return result; } } diff --git a/mobile/lib/db/remote/migration.dart b/mobile/lib/db/remote/migration.dart index af409d9223..db4d95f2b6 100644 --- a/mobile/lib/db/remote/migration.dart +++ b/mobile/lib/db/remote/migration.dart @@ -1,6 +1,15 @@ import "package:flutter/cupertino.dart"; import "package:sqlite_async/sqlite_async.dart"; +const collectionColumns = + 'id, owner, enc_key, enc_key_nonce, name, type, local_path, ' + 'is_deleted, updation_time, sharees, public_urls, mmd_encoded_json, ' + 'mmd_ver, pub_mmd_encoded_json, pub_mmd_ver, shared_mmd_json, ' + 'shared_mmd_ver'; + +String collectionValuePlaceHolder = + collectionColumns.split(',').map((_) => '?').join(','); + class RemoteDBMigration { static const migrationScripts = [ ''' @@ -12,31 +21,31 @@ class RemoteDBMigration { name TEXT NOT NULL, type TEXT NOT NULL, local_path TEXT, - is_deleted INTEGER NOT NULL + is_deleted INTEGER NOT NULL, updation_time INTEGER NOT NULL, - sharees TEXT NOT NULL DEFAULT [], - public_urls TEXT NOT NULL DEFAULT [], - mmd_encoded_json TEXT NOT NULL DEFAULT {}, + sharees TEXT NOT NULL DEFAULT '[]', + public_urls TEXT NOT NULL DEFAULT '[]', + mmd_encoded_json TEXT NOT NULL DEFAULT '{}', mmd_ver INTEGER NOT NULL DEFAULT 0, - pub_mmd_encoded_json TEXT DEFAULT {}', + pub_mmd_encoded_json TEXT DEFAULT '{}', pub_mmd_ver INTEGER NOT NULL DEFAULT 0, - shared_mmd_json TEXT NOT NULL {}, - shared_mmd_ver INTEGER NOT NULL DEFAULT 0, - ) - ''', - ''' - CREATE TABLE collection_files ( - file_id INTEGER NOT NULL, - collection_id INTEGER NOT NULL, - PRIMARY KEY (file_id, collection_id) - enc_key TEXT NOT NULL, - enc_key_nonce TEXT NOT NULL, - is_deleted INTEGER NOT NULL - updated_at INTEGER NOT NULL, - created_at INTEGER NOT NULL DEFAULT 0, - ) + shared_mmd_json TEXT NOT NULL DEFAULT '{}', + shared_mmd_ver INTEGER NOT NULL DEFAULT 0 + ); ''', ]; + // ''' + // CREATE TABLE collection_files ( + // file_id INTEGER NOT NULL, + // collection_id INTEGER NOT NULL, + // PRIMARY KEY (file_id, collection_id) + // enc_key TEXT NOT NULL, + // enc_key_nonce TEXT NOT NULL, + // is_deleted INTEGER NOT NULL + // updated_at INTEGER NOT NULL, + // created_at INTEGER NOT NULL DEFAULT 0, + // ) + // ''', static Future migrate( SqliteDatabase database, diff --git a/mobile/lib/models/collection/collection.dart b/mobile/lib/models/collection/collection.dart index 1ab6252f9c..4f2402577c 100644 --- a/mobile/lib/models/collection/collection.dart +++ b/mobile/lib/models/collection/collection.dart @@ -238,15 +238,13 @@ class Collection { final sharees = List.from( (json.decode(map['sharees']) as List).map((x) => User.fromMap(x)), ); - final List publicURLs = map['public_urls'] == null - ? [] - : List.from( - (json.decode(map['public_urls']) as List) - .map((x) => PublicURL.fromMap(x)), - ); + final List publicURLs = List.from( + (json.decode(map['public_urls']) as List) + .map((x) => PublicURL.fromMap(x)), + ); return Collection( id: map['id'], - owner: User.fromMap(map['owner']), + owner: User.fromJson(map['owner']), encryptedKey: map['enc_key'], keyDecryptionNonce: map['enc_key_nonce'], name: map['name'], @@ -255,36 +253,36 @@ class Collection { publicURLs: publicURLs, updationTime: map['updation_time'], localPath: map['local_path'], - isDeleted: map['isDeleted'] ?? false, - mMdEncodedJson: map['mmd_encoded_json'] ?? '{}', - mMdVersion: map['mmd_ver'] ?? 0, - mMdPubEncodedJson: map['pub_mmd_encoded_json'] ?? '{}', - mMbPubVersion: map['pub_mmd_ver'] ?? 0, - sharedMmdJson: map['shared_mmd_json'] ?? '{}', - sharedMmdVersion: map['shared_mmd_ver'] ?? 0, + isDeleted: (map['is_deleted'] as int) == 1, + mMdEncodedJson: map['mmd_encoded_json'], + mMdVersion: map['mmd_ver'], + mMdPubEncodedJson: map['pub_mmd_encoded_json'], + mMbPubVersion: map['pub_mmd_ver'], + sharedMmdJson: map['shared_mmd_json'], + sharedMmdVersion: map['shared_mmd_ver'], ); } - static Map toRow(Collection c) { - return { - 'id': c.id, - 'owner': c.owner.toMap(), - 'enc_key': c.encryptedKey, - 'enc_key_nonce': c.keyDecryptionNonce, - 'name': c.name, - 'type': typeToString(c.type), - 'sharees': json.encode(c.sharees.map((x) => x.toMap()).toList()), - 'public_urls': json.encode(c.publicURLs.map((x) => x.toMap()).toList()), - 'updation_time': c.updationTime, - 'local_path': c.localPath, - 'isDeleted': c.isDeleted, - 'mmd_encoded_json': c.mMdEncodedJson, - 'mmd_ver': c.mMdVersion, - 'pub_mmd_encoded_json': c.mMdPubEncodedJson, - 'pub_mmd_ver': c.mMbPubVersion, - 'shared_mmd_json': c.sharedMmdJson, - 'shared_mmd_ver': c.sharedMmdVersion, - }; + List rowValiues() { + return [ + id, + owner.toJson(), + encryptedKey, + keyDecryptionNonce, + name, + typeToString(type), + localPath, + isDeleted ? 1 : 0, + updationTime, + json.encode(sharees.map((x) => x.toMap()).toList()), + json.encode(publicURLs.map((x) => x.toMap()).toList()), + mMdEncodedJson, + mMdVersion, + mMdPubEncodedJson, + mMbPubVersion, + sharedMmdJson, + sharedMmdVersion, + ]; } } diff --git a/mobile/lib/service_locator.dart b/mobile/lib/service_locator.dart index d33bef747e..6d284ba298 100644 --- a/mobile/lib/service_locator.dart +++ b/mobile/lib/service_locator.dart @@ -3,6 +3,7 @@ import "package:ente_cast/ente_cast.dart"; import "package:ente_cast_normal/ente_cast_normal.dart"; import "package:ente_feature_flag/ente_feature_flag.dart"; import "package:package_info_plus/package_info_plus.dart"; +import "package:photos/db/remote/db.dart"; import "package:photos/gateways/entity_gw.dart"; import "package:photos/services/billing_service.dart"; import "package:photos/services/entity_service.dart"; @@ -137,3 +138,9 @@ FaceRecognitionService get faceRecognitionService { _faceRecognitionService ??= FaceRecognitionService(); return _faceRecognitionService!; } + +RemoteDB? _remoteDB; +RemoteDB get remoteDB { + _remoteDB ??= RemoteDB(); + return _remoteDB!; +} diff --git a/mobile/lib/services/collections_service.dart b/mobile/lib/services/collections_service.dart index 7528dda73d..7ce1db2d4d 100644 --- a/mobile/lib/services/collections_service.dart +++ b/mobile/lib/services/collections_service.dart @@ -16,6 +16,7 @@ import 'package:photos/core/network/network.dart'; import 'package:photos/db/collections_db.dart'; import 'package:photos/db/device_files_db.dart'; import 'package:photos/db/files_db.dart'; +import "package:photos/db/remote/db.dart"; import 'package:photos/db/trash_db.dart'; import 'package:photos/events/collection_updated_event.dart'; import 'package:photos/events/files_updated_event.dart'; @@ -47,8 +48,8 @@ import "package:photos/utils/local_settings.dart"; import 'package:shared_preferences/shared_preferences.dart'; class CollectionsService { - static const _collectionSyncTimeKeyPrefix = "collection_sync_time_"; - static const _collectionsSyncTimeKey = "collections_sync_time_x"; + static const _collectionSyncTimeKeyPrefix = "collection_sync_time_x"; + static const _collectionsSyncTimeKey = "collections_sync_time_xx"; static const int kMaximumWriteAttempts = 5; @@ -87,13 +88,25 @@ class CollectionsService { Future init(SharedPreferences preferences) async { _prefs = preferences; - final collections = await _db.getAllCollections(); + await remoteDB.init(); + final collections = await _db.getAllCollections(); + final List collectionsToInsert = []; for (final collection in collections) { // using deprecated method because the path is stored in encrypted // format in the DB - _cacheCollectionAttributes(collection); + final r = _cacheCollectionAttributes(collection); + collectionsToInsert.add(r); } + await remoteDB.insertCollections(collectionsToInsert); + await _db.clearTable(); + if (collectionsToInsert.isEmpty) { + final newColections = await remoteDB.getAllCollections(); + for (final collection in newColections) { + _cacheLocalPathAndCollection(collection); + } + } + Bus.instance.on().listen((event) { _collectionIDToNewestFileTime = null; if (event.collectionID != null) { @@ -131,6 +144,7 @@ class CollectionsService { int maxUpdationTime = lastCollectionUpdationTime; final ownerID = _config.getUserID(); bool shouldFireDeleteEvent = false; + final Set toDelete = {}; for (final collection in fetchedCollections) { if (collection.isDeleted) { await _filesDB.deleteCollection(collection.id); @@ -141,7 +155,7 @@ class CollectionsService { } // remove reference for incoming collections when unshared/deleted if (collection.isDeleted && ownerID != collection.owner.id) { - await _db.deleteCollection(collection.id); + toDelete.add(collection.id); } else { // keep entry for deletedCollection as collectionKey may be used during // trash file decryption @@ -159,6 +173,9 @@ class CollectionsService { ), ); } + if (toDelete.isNotEmpty) { + await remoteDB.deleteEntries(toDelete, RemoteTable.collections); + } await _updateDB(updatedCollections); await _prefs.setInt(_collectionsSyncTimeKey, maxUpdationTime); watch.logAndReset("till DB insertion ${updatedCollections.length}"); @@ -191,8 +208,12 @@ class CollectionsService { } Future> getCollectionIDsToBeSynced() async { - final idsToRemoveUpdateTimeMap = - await _db.getActiveIDsAndRemoteUpdateTime(); + final idsToRemoveUpdateTimeMap = {}; + for (final collection in _collectionIDToCollections.values) { + if (!collection.isDeleted) { + idsToRemoveUpdateTimeMap[collection.id] = collection.updationTime; + } + } final result = {}; for (final MapEntry e in idsToRemoveUpdateTimeMap.entries) { final int cid = e.key; @@ -551,7 +572,9 @@ class CollectionsService { } _collectionIDToCollections[collectionID] = _collectionIDToCollections[collectionID]!.copyWith(sharees: sharees); - unawaited(_db.insert([_collectionIDToCollections[collectionID]!])); + unawaited( + _updateDB([_collectionIDToCollections[collectionID]!]), + ); RemoteSyncService.instance.sync(silently: true).ignore(); return sharees; } on DioException catch (e) { @@ -577,7 +600,9 @@ class CollectionsService { } _collectionIDToCollections[collectionID] = _collectionIDToCollections[collectionID]!.copyWith(sharees: sharees); - unawaited(_db.insert([_collectionIDToCollections[collectionID]!])); + unawaited( + _updateDB([_collectionIDToCollections[collectionID]!]), + ); RemoteSyncService.instance.sync(silently: true).ignore(); return sharees; } catch (e) { @@ -638,7 +663,7 @@ class CollectionsService { if (isBulkDelete) { final deletedCollection = collection.copyWith(isDeleted: true); _collectionIDToCollections[collection.id] = deletedCollection; - unawaited(_db.insert([deletedCollection])); + unawaited(_updateDB([deletedCollection])); } else { await _handleCollectionDeletion(collection); } @@ -656,7 +681,7 @@ class CollectionsService { Future _handleCollectionDeletion(Collection collection) async { await _filesDB.deleteCollection(collection.id); final deletedCollection = collection.copyWith(isDeleted: true); - unawaited(_db.insert([deletedCollection])); + unawaited(_updateDB([deletedCollection])); _collectionIDToCollections[collection.id] = deletedCollection; Bus.instance.fire( CollectionUpdatedEvent( @@ -962,7 +987,7 @@ class CollectionsService { }, ); collection.publicURLs.add(PublicURL.fromMap(response.data["result"])); - await _db.insert(List.from([collection])); + await _updateDB(List.from([collection])); _collectionIDToCollections[collection.id] = collection; Bus.instance.fire( CollectionUpdatedEvent(collection.id, [], "shareUrL"), @@ -991,7 +1016,7 @@ class CollectionsService { // remove existing url information collection.publicURLs.clear(); collection.publicURLs.add(PublicURL.fromMap(response.data["result"])); - await _db.insert(List.from([collection])); + await _updateDB(List.from([collection])); _collectionIDToCollections[collection.id] = collection; Bus.instance.fire( CollectionUpdatedEvent(collection.id, [], "updateUrl"), @@ -1013,7 +1038,7 @@ class CollectionsService { "/collections/share-url/" + collection.id.toString(), ); collection.publicURLs.clear(); - await _db.insert(List.from([collection])); + await _updateDB(List.from([collection])); _collectionIDToCollections[collection.id] = collection; Bus.instance.fire( CollectionUpdatedEvent( @@ -1325,7 +1350,7 @@ class CollectionsService { assert(response.data != null); final collectionData = response.data["collection"]; final collection = await _fromRemoteCollection(collectionData); - await _db.insert(List.from([collection])); + await _updateDB(List.from([collection])); _cacheLocalPathAndCollection(collection); return collection; } catch (e) { @@ -1902,19 +1927,6 @@ class CollectionsService { }); } - @Deprecated("Use _cacheLocalPathAndCollection instead") - CollectionV2 _cacheCollectionAttributes(CollectionV2 collection) { - final String decryptedName = _getDecryptedCollectionName(collection); - collection.setName(decryptedName); - if (collection.canLinkToDevicePath(_config.getUserID()!)) { - collection.decryptedPath = _decryptCollectionPath(collection); - _localPathToCollectionID[collection.decryptedPath!] = collection.id; - } - final Collection c = Collection.fromOldCollection(collection); - _collectionIDToCollections[collection.id] = c; - return collection; - } - Collection _cacheLocalPathAndCollection(Collection collection) { assert( collection.name != null, @@ -1927,6 +1939,23 @@ class CollectionsService { return collection; } + bool hasSyncedCollections() { + return _prefs.containsKey(_collectionsSyncTimeKey); + } + + @Deprecated("Use _cacheLocalPathAndCollection instead") + Collection _cacheCollectionAttributes(CollectionV2 collection) { + final String decryptedName = _getDecryptedCollectionName(collection); + collection.setName(decryptedName); + if (collection.canLinkToDevicePath(_config.getUserID()!)) { + collection.decryptedPath = _decryptCollectionPath(collection); + _localPathToCollectionID[collection.decryptedPath!] = collection.id; + } + final Collection c = Collection.fromOldCollection(collection); + _collectionIDToCollections[collection.id] = c; + return c; + } + String _decryptCollectionPath(CollectionV2 collection) { final existingPath = collection.decryptedPath; if (existingPath != null && existingPath.isNotEmpty) { @@ -1950,10 +1979,6 @@ class CollectionsService { ); } - bool hasSyncedCollections() { - return _prefs.containsKey(_collectionsSyncTimeKey); - } - String _getDecryptedCollectionName(CollectionV2 collection) { if (collection.isDeleted) { return "Deleted Album"; @@ -1990,7 +2015,7 @@ class CollectionsService { return; } try { - await _db.insert(collections); + await remoteDB.insertCollections(collections); } catch (e) { _logger.severe("Failed to update collections", e); if (attempt < kMaximumWriteAttempts) {