[mob] Use remote db for collections

2025-02-14 15:24:57 +05:30
parent 6e85d24286
commit 2a1f2aded1
6 changed files with 202 additions and 99 deletions
--- a/mobile/lib/db/collections_db.dart
+++ b/mobile/lib/db/collections_db.dart
@@ -10,6 +10,7 @@ import "package:photos/models/collection/collection_old.dart";
 import 'package:sqflite/sqflite.dart';
 import 'package:sqflite_migration/sqflite_migration.dart';

+@Deprecated("Use remoteDB instead") 
 class CollectionsDB {
  static const _databaseName = "ente.collections.db";
  static const table = 'collections';
--- a/mobile/lib/db/remote/db.dart
+++ b/mobile/lib/db/remote/db.dart
@@ -1,32 +1,95 @@
 import "dart:developer";
 import "dart:io";

+import "package:collection/collection.dart";
+import "package:flutter/foundation.dart";
 import "package:path/path.dart";
 import "package:path_provider/path_provider.dart";
 import "package:photos/db/remote/migration.dart";
+import "package:photos/models/collection/collection.dart";
 import "package:sqlite_async/sqlite_async.dart";

 var devLog = log;

+// ignore: constant_identifier_names
+enum RemoteTable { collections, collection_files, files, entities }
+
 class RemoteDB {
  static const _databaseName = "remote.db";
-  // only have a single app-wide reference to the database
-  static Future<SqliteDatabase>? _sqliteAsyncDBFuture;
+  static const _batchInsertMaxCount = 1000;
+  late final SqliteDatabase _sqliteDB;

-  Future<SqliteDatabase> get sqliteAsyncDB async {
-    // lazily instantiate the db the first time it is accessed
-    _sqliteAsyncDBFuture ??= _initSqliteAsyncDatabase();
-    return _sqliteAsyncDBFuture!;
-  }
-
-  // this opens the database (and creates it if it doesn't exist)
-  Future<SqliteDatabase> _initSqliteAsyncDatabase() async {
+  Future<void> init() async {
+    devLog("Starting RemoteDB init");
    final Directory documentsDirectory =
        await getApplicationDocumentsDirectory();
    final String path = join(documentsDirectory.path, _databaseName);
-    devLog("DB path " + path);
+
    final database = SqliteDatabase(path: path);
    await RemoteDBMigration.migrate(database);
-    return database;
+    _sqliteDB = database;
+    devLog("RemoteDB init complete $path");
+  }
+
+  Future<List<Collection>> getAllCollections() async {
+    final result = <Collection>[];
+    final cursor = await _sqliteDB.getAll("SELECT * FROM collections");
+    for (final row in cursor) {
+      result.add(Collection.fromRow(row));
+    }
+    return result;
+  }
+
+  Future<void> insertCollections(List<Collection> collections) async {
+    if (collections.isEmpty) return;
+    final stopwatch = Stopwatch()..start();
+    int inserted = 0;
+    await Future.forEach(collections.slices(_batchInsertMaxCount),
+        (slice) async {
+      debugPrint(
+        '$runtimeType insertCollections inserting slice of [${inserted + 1}, ${inserted + slice.length}] entries',
+      );
+      final List<List<Object?>> values =
+          slice.map((e) => e.rowValiues()).toList();
+      await _sqliteDB.executeBatch(
+        'INSERT OR REPLACE INTO collections ($collectionColumns) values($collectionValuePlaceHolder)',
+        values,
+      );
+      inserted += slice.length;
+    });
+    debugPrint(
+      '$runtimeType insertCollections complete in ${stopwatch.elapsed.inMilliseconds}ms for ${collections.length} collections',
+    );
+  }
+
+  Future<void> deleteEntries<T>(Set<T> ids, RemoteTable table) async {
+    if (ids.isEmpty) return;
+    final stopwatch = Stopwatch()..start();
+    await _sqliteDB.execute(
+      'DELETE FROM ${table.name.toLowerCase()} WHERE id IN (${ids.join(',')})',
+    );
+    debugPrint(
+      '$runtimeType deleteEntries complete in ${stopwatch.elapsed.inMilliseconds}ms for ${ids.length} $table entries',
+    );
+  }
+
+  Future<Set<T>> _getByIds<T>(
+    Set<int> ids,
+    String table,
+    T Function(
+      Map<String, Object?> row,
+    ) mapRow, {
+    String columnName = "id",
+  }) async {
+    final result = <T>{};
+    if (ids.isNotEmpty) {
+      final rows = await _sqliteDB.getAll(
+        'SELECT * from $table where $columnName IN (${ids.join(',')})',
+      );
+      for (final row in rows) {
+        result.add(mapRow(row));
+      }
+    }
+    return result;
  }
 }
--- a/mobile/lib/db/remote/migration.dart
+++ b/mobile/lib/db/remote/migration.dart
@@ -1,6 +1,15 @@
 import "package:flutter/cupertino.dart";
 import "package:sqlite_async/sqlite_async.dart";

+const collectionColumns =
+    'id, owner, enc_key, enc_key_nonce, name, type, local_path, '
+    'is_deleted, updation_time, sharees, public_urls, mmd_encoded_json, '
+    'mmd_ver, pub_mmd_encoded_json, pub_mmd_ver, shared_mmd_json, '
+    'shared_mmd_ver';
+
+String collectionValuePlaceHolder =
+    collectionColumns.split(',').map((_) => '?').join(',');
+
 class RemoteDBMigration {
  static const migrationScripts = [
    '''
@@ -12,31 +21,31 @@ class RemoteDBMigration {
      name TEXT NOT NULL,
      type TEXT NOT NULL,
      local_path TEXT,
-      is_deleted INTEGER NOT NULL
+      is_deleted INTEGER NOT NULL,
      updation_time INTEGER NOT NULL,
-      sharees TEXT NOT NULL DEFAULT [],
-      public_urls TEXT NOT NULL DEFAULT [],
-      mmd_encoded_json TEXT NOT NULL DEFAULT {},
+      sharees TEXT NOT NULL DEFAULT '[]',
+      public_urls TEXT NOT NULL DEFAULT '[]',
+      mmd_encoded_json TEXT NOT NULL DEFAULT '{}',
      mmd_ver INTEGER NOT NULL DEFAULT 0,
-      pub_mmd_encoded_json TEXT DEFAULT {}',
+      pub_mmd_encoded_json TEXT DEFAULT '{}',
      pub_mmd_ver INTEGER NOT NULL DEFAULT 0,
-      shared_mmd_json TEXT NOT NULL {},
-      shared_mmd_ver INTEGER NOT NULL DEFAULT 0,
-    )
-    ''',
-    '''
-    CREATE TABLE collection_files (
-      file_id INTEGER NOT NULL,
-      collection_id INTEGER NOT NULL,
-      PRIMARY KEY (file_id, collection_id)
-      enc_key TEXT NOT NULL,
-      enc_key_nonce TEXT NOT NULL,
-      is_deleted INTEGER NOT NULL
-      updated_at INTEGER NOT NULL,
-      created_at INTEGER NOT NULL DEFAULT 0,
-    )
+      shared_mmd_json TEXT NOT NULL DEFAULT '{}',
+      shared_mmd_ver INTEGER NOT NULL DEFAULT 0
+    );
    ''',
  ];
+  // '''
+  //   CREATE TABLE collection_files (
+  //     file_id INTEGER NOT NULL,
+  //     collection_id INTEGER NOT NULL,
+  //     PRIMARY KEY (file_id, collection_id)
+  //     enc_key TEXT NOT NULL,
+  //     enc_key_nonce TEXT NOT NULL,
+  //     is_deleted INTEGER NOT NULL
+  //     updated_at INTEGER NOT NULL,
+  //     created_at INTEGER NOT NULL DEFAULT 0,
+  //   )
+  //   ''',

  static Future<void> migrate(
    SqliteDatabase database,
--- a/mobile/lib/models/collection/collection.dart
+++ b/mobile/lib/models/collection/collection.dart
@@ -238,15 +238,13 @@ class Collection {
    final sharees = List<User>.from(
      (json.decode(map['sharees']) as List).map((x) => User.fromMap(x)),
    );
-    final List<PublicURL> publicURLs = map['public_urls'] == null
-        ? []
-        : List<PublicURL>.from(
-            (json.decode(map['public_urls']) as List)
-                .map((x) => PublicURL.fromMap(x)),
-          );
+    final List<PublicURL> publicURLs = List<PublicURL>.from(
+      (json.decode(map['public_urls']) as List)
+          .map((x) => PublicURL.fromMap(x)),
+    );
    return Collection(
      id: map['id'],
-      owner: User.fromMap(map['owner']),
+      owner: User.fromJson(map['owner']),
      encryptedKey: map['enc_key'],
      keyDecryptionNonce: map['enc_key_nonce'],
      name: map['name'],
@@ -255,36 +253,36 @@ class Collection {
      publicURLs: publicURLs,
      updationTime: map['updation_time'],
      localPath: map['local_path'],
-      isDeleted: map['isDeleted'] ?? false,
-      mMdEncodedJson: map['mmd_encoded_json'] ?? '{}',
-      mMdVersion: map['mmd_ver'] ?? 0,
-      mMdPubEncodedJson: map['pub_mmd_encoded_json'] ?? '{}',
-      mMbPubVersion: map['pub_mmd_ver'] ?? 0,
-      sharedMmdJson: map['shared_mmd_json'] ?? '{}',
-      sharedMmdVersion: map['shared_mmd_ver'] ?? 0,
+      isDeleted: (map['is_deleted'] as int) == 1,
+      mMdEncodedJson: map['mmd_encoded_json'],
+      mMdVersion: map['mmd_ver'],
+      mMdPubEncodedJson: map['pub_mmd_encoded_json'],
+      mMbPubVersion: map['pub_mmd_ver'],
+      sharedMmdJson: map['shared_mmd_json'],
+      sharedMmdVersion: map['shared_mmd_ver'],
    );
  }

-  static Map<String, dynamic> toRow(Collection c) {
-    return {
-      'id': c.id,
-      'owner': c.owner.toMap(),
-      'enc_key': c.encryptedKey,
-      'enc_key_nonce': c.keyDecryptionNonce,
-      'name': c.name,
-      'type': typeToString(c.type),
-      'sharees': json.encode(c.sharees.map((x) => x.toMap()).toList()),
-      'public_urls': json.encode(c.publicURLs.map((x) => x.toMap()).toList()),
-      'updation_time': c.updationTime,
-      'local_path': c.localPath,
-      'isDeleted': c.isDeleted,
-      'mmd_encoded_json': c.mMdEncodedJson,
-      'mmd_ver': c.mMdVersion,
-      'pub_mmd_encoded_json': c.mMdPubEncodedJson,
-      'pub_mmd_ver': c.mMbPubVersion,
-      'shared_mmd_json': c.sharedMmdJson,
-      'shared_mmd_ver': c.sharedMmdVersion,
-    };
+  List<Object?> rowValiues() {
+    return [
+      id,
+      owner.toJson(),
+      encryptedKey,
+      keyDecryptionNonce,
+      name,
+      typeToString(type),
+      localPath,
+      isDeleted ? 1 : 0,
+      updationTime,
+      json.encode(sharees.map((x) => x.toMap()).toList()),
+      json.encode(publicURLs.map((x) => x.toMap()).toList()),
+      mMdEncodedJson,
+      mMdVersion,
+      mMdPubEncodedJson,
+      mMbPubVersion,
+      sharedMmdJson,
+      sharedMmdVersion,
+    ];
  }
 }

--- a/mobile/lib/service_locator.dart
+++ b/mobile/lib/service_locator.dart
@@ -3,6 +3,7 @@ import "package:ente_cast/ente_cast.dart";
 import "package:ente_cast_normal/ente_cast_normal.dart";
 import "package:ente_feature_flag/ente_feature_flag.dart";
 import "package:package_info_plus/package_info_plus.dart";
+import "package:photos/db/remote/db.dart";
 import "package:photos/gateways/entity_gw.dart";
 import "package:photos/services/billing_service.dart";
 import "package:photos/services/entity_service.dart";
@@ -137,3 +138,9 @@ FaceRecognitionService get faceRecognitionService {
  _faceRecognitionService ??= FaceRecognitionService();
  return _faceRecognitionService!;
 }
+
+RemoteDB? _remoteDB;
+RemoteDB get remoteDB {
+  _remoteDB ??= RemoteDB();
+  return _remoteDB!;
+}
--- a/mobile/lib/services/collections_service.dart
+++ b/mobile/lib/services/collections_service.dart
@@ -16,6 +16,7 @@ import 'package:photos/core/network/network.dart';
 import 'package:photos/db/collections_db.dart';
 import 'package:photos/db/device_files_db.dart';
 import 'package:photos/db/files_db.dart';
+import "package:photos/db/remote/db.dart";
 import 'package:photos/db/trash_db.dart';
 import 'package:photos/events/collection_updated_event.dart';
 import 'package:photos/events/files_updated_event.dart';
@@ -47,8 +48,8 @@ import "package:photos/utils/local_settings.dart";
 import 'package:shared_preferences/shared_preferences.dart';

 class CollectionsService {
-  static const _collectionSyncTimeKeyPrefix = "collection_sync_time_";
-  static const _collectionsSyncTimeKey = "collections_sync_time_x";
+  static const _collectionSyncTimeKeyPrefix = "collection_sync_time_x";
+  static const _collectionsSyncTimeKey = "collections_sync_time_xx";

  static const int kMaximumWriteAttempts = 5;

@@ -87,13 +88,25 @@ class CollectionsService {

  Future<void> init(SharedPreferences preferences) async {
    _prefs = preferences;
-    final collections = await _db.getAllCollections();
+    await remoteDB.init();

+    final collections = await _db.getAllCollections();
+    final List<Collection> collectionsToInsert = [];
    for (final collection in collections) {
      // using deprecated method because the path is stored in encrypted
      // format in the DB
-      _cacheCollectionAttributes(collection);
+      final r = _cacheCollectionAttributes(collection);
+      collectionsToInsert.add(r);
    }
+    await remoteDB.insertCollections(collectionsToInsert);
+    await _db.clearTable();
+    if (collectionsToInsert.isEmpty) {
+      final newColections = await remoteDB.getAllCollections();
+      for (final collection in newColections) {
+        _cacheLocalPathAndCollection(collection);
+      }
+    }
+
    Bus.instance.on<CollectionUpdatedEvent>().listen((event) {
      _collectionIDToNewestFileTime = null;
      if (event.collectionID != null) {
@@ -131,6 +144,7 @@ class CollectionsService {
    int maxUpdationTime = lastCollectionUpdationTime;
    final ownerID = _config.getUserID();
    bool shouldFireDeleteEvent = false;
+    final Set<int> toDelete = {};
    for (final collection in fetchedCollections) {
      if (collection.isDeleted) {
        await _filesDB.deleteCollection(collection.id);
@@ -141,7 +155,7 @@ class CollectionsService {
      }
      // remove reference for incoming collections when unshared/deleted
      if (collection.isDeleted && ownerID != collection.owner.id) {
-        await _db.deleteCollection(collection.id);
+        toDelete.add(collection.id);
      } else {
        // keep entry for deletedCollection as collectionKey may be used during
        // trash file decryption
@@ -159,6 +173,9 @@ class CollectionsService {
        ),
      );
    }
+    if (toDelete.isNotEmpty) {
+      await remoteDB.deleteEntries(toDelete, RemoteTable.collections);
+    }
    await _updateDB(updatedCollections);
    await _prefs.setInt(_collectionsSyncTimeKey, maxUpdationTime);
    watch.logAndReset("till DB insertion ${updatedCollections.length}");
@@ -191,8 +208,12 @@ class CollectionsService {
  }

  Future<Map<int, int>> getCollectionIDsToBeSynced() async {
-    final idsToRemoveUpdateTimeMap =
-        await _db.getActiveIDsAndRemoteUpdateTime();
+    final idsToRemoveUpdateTimeMap = <int, int>{};
+    for (final collection in _collectionIDToCollections.values) {
+      if (!collection.isDeleted) {
+        idsToRemoveUpdateTimeMap[collection.id] = collection.updationTime;
+      }
+    }
    final result = <int, int>{};
    for (final MapEntry<int, int> e in idsToRemoveUpdateTimeMap.entries) {
      final int cid = e.key;
@@ -551,7 +572,9 @@ class CollectionsService {
      }
      _collectionIDToCollections[collectionID] =
          _collectionIDToCollections[collectionID]!.copyWith(sharees: sharees);
-      unawaited(_db.insert([_collectionIDToCollections[collectionID]!]));
+      unawaited(
+        _updateDB([_collectionIDToCollections[collectionID]!]),
+      );
      RemoteSyncService.instance.sync(silently: true).ignore();
      return sharees;
    } on DioException catch (e) {
@@ -577,7 +600,9 @@ class CollectionsService {
      }
      _collectionIDToCollections[collectionID] =
          _collectionIDToCollections[collectionID]!.copyWith(sharees: sharees);
-      unawaited(_db.insert([_collectionIDToCollections[collectionID]!]));
+      unawaited(
+        _updateDB([_collectionIDToCollections[collectionID]!]),
+      );
      RemoteSyncService.instance.sync(silently: true).ignore();
      return sharees;
    } catch (e) {
@@ -638,7 +663,7 @@ class CollectionsService {
      if (isBulkDelete) {
        final deletedCollection = collection.copyWith(isDeleted: true);
        _collectionIDToCollections[collection.id] = deletedCollection;
-        unawaited(_db.insert([deletedCollection]));
+        unawaited(_updateDB([deletedCollection]));
      } else {
        await _handleCollectionDeletion(collection);
      }
@@ -656,7 +681,7 @@ class CollectionsService {
  Future<void> _handleCollectionDeletion(Collection collection) async {
    await _filesDB.deleteCollection(collection.id);
    final deletedCollection = collection.copyWith(isDeleted: true);
-    unawaited(_db.insert([deletedCollection]));
+    unawaited(_updateDB([deletedCollection]));
    _collectionIDToCollections[collection.id] = deletedCollection;
    Bus.instance.fire(
      CollectionUpdatedEvent(
@@ -962,7 +987,7 @@ class CollectionsService {
        },
      );
      collection.publicURLs.add(PublicURL.fromMap(response.data["result"]));
-      await _db.insert(List.from([collection]));
+      await _updateDB(List.from([collection]));
      _collectionIDToCollections[collection.id] = collection;
      Bus.instance.fire(
        CollectionUpdatedEvent(collection.id, <EnteFile>[], "shareUrL"),
@@ -991,7 +1016,7 @@ class CollectionsService {
      // remove existing url information
      collection.publicURLs.clear();
      collection.publicURLs.add(PublicURL.fromMap(response.data["result"]));
-      await _db.insert(List.from([collection]));
+      await _updateDB(List.from([collection]));
      _collectionIDToCollections[collection.id] = collection;
      Bus.instance.fire(
        CollectionUpdatedEvent(collection.id, <EnteFile>[], "updateUrl"),
@@ -1013,7 +1038,7 @@ class CollectionsService {
        "/collections/share-url/" + collection.id.toString(),
      );
      collection.publicURLs.clear();
-      await _db.insert(List.from([collection]));
+      await _updateDB(List.from([collection]));
      _collectionIDToCollections[collection.id] = collection;
      Bus.instance.fire(
        CollectionUpdatedEvent(
@@ -1325,7 +1350,7 @@ class CollectionsService {
      assert(response.data != null);
      final collectionData = response.data["collection"];
      final collection = await _fromRemoteCollection(collectionData);
-      await _db.insert(List.from([collection]));
+      await _updateDB(List.from([collection]));
      _cacheLocalPathAndCollection(collection);
      return collection;
    } catch (e) {
@@ -1902,19 +1927,6 @@ class CollectionsService {
    });
  }

-  @Deprecated("Use _cacheLocalPathAndCollection instead")
-  CollectionV2 _cacheCollectionAttributes(CollectionV2 collection) {
-    final String decryptedName = _getDecryptedCollectionName(collection);
-    collection.setName(decryptedName);
-    if (collection.canLinkToDevicePath(_config.getUserID()!)) {
-      collection.decryptedPath = _decryptCollectionPath(collection);
-      _localPathToCollectionID[collection.decryptedPath!] = collection.id;
-    }
-    final Collection c = Collection.fromOldCollection(collection);
-    _collectionIDToCollections[collection.id] = c;
-    return collection;
-  }
-
  Collection _cacheLocalPathAndCollection(Collection collection) {
    assert(
      collection.name != null,
@@ -1927,6 +1939,23 @@ class CollectionsService {
    return collection;
  }

+  bool hasSyncedCollections() {
+    return _prefs.containsKey(_collectionsSyncTimeKey);
+  }
+
+  @Deprecated("Use _cacheLocalPathAndCollection instead")
+  Collection _cacheCollectionAttributes(CollectionV2 collection) {
+    final String decryptedName = _getDecryptedCollectionName(collection);
+    collection.setName(decryptedName);
+    if (collection.canLinkToDevicePath(_config.getUserID()!)) {
+      collection.decryptedPath = _decryptCollectionPath(collection);
+      _localPathToCollectionID[collection.decryptedPath!] = collection.id;
+    }
+    final Collection c = Collection.fromOldCollection(collection);
+    _collectionIDToCollections[collection.id] = c;
+    return c;
+  }
+
  String _decryptCollectionPath(CollectionV2 collection) {
    final existingPath = collection.decryptedPath;
    if (existingPath != null && existingPath.isNotEmpty) {
@@ -1950,10 +1979,6 @@ class CollectionsService {
    );
  }

-  bool hasSyncedCollections() {
-    return _prefs.containsKey(_collectionsSyncTimeKey);
-  }
-
  String _getDecryptedCollectionName(CollectionV2 collection) {
    if (collection.isDeleted) {
      return "Deleted Album";
@@ -1990,7 +2015,7 @@ class CollectionsService {
      return;
    }
    try {
-      await _db.insert(collections);
+      await remoteDB.insertCollections(collections);
    } catch (e) {
      _logger.severe("Failed to update collections", e);
      if (attempt < kMaximumWriteAttempts) {