From cbff68bc42e44ca4d32f535220c2e2aaa06acd8f Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Fri, 28 Feb 2025 10:56:15 +0530 Subject: [PATCH] [server] Persist remote pull response --- mobile/lib/db/remote/db.dart | 21 ++++- mobile/lib/db/remote/migration.dart | 19 ++-- mobile/lib/models/api/diff/diff.dart | 50 ++++++++++- mobile/lib/services/collections_service.dart | 18 ++-- mobile/lib/services/diff/remote_pull.dart | 92 +++++++++++++++++++- mobile/lib/services/remote_sync_service.dart | 8 +- 6 files changed, 183 insertions(+), 25 deletions(-) diff --git a/mobile/lib/db/remote/db.dart b/mobile/lib/db/remote/db.dart index 1c80f3252b..4c00247ad0 100644 --- a/mobile/lib/db/remote/db.dart +++ b/mobile/lib/db/remote/db.dart @@ -16,7 +16,7 @@ var devLog = log; enum RemoteTable { collections, collection_files, files, entities } class RemoteDB { - static const _databaseName = "remote.db"; + static const _databaseName = "remotex2.db"; static const _batchInsertMaxCount = 1000; late final SqliteDatabase _sqliteDB; @@ -82,13 +82,30 @@ class RemoteDB { 'INSERT OR REPLACE INTO collection_files ($collectionFilesColumns) values(?, ?, ?, ?, ?, ?, ?)', values, ); + final List> fileValues = slice + .map( + (e) => [ + e.fileItem.fileID, + e.fileItem.ownerID, + e.fileItem.fileDecryotionHeader, + e.fileItem.thumnailDecryptionHeader, + e.fileItem.metadata?.toEncodedJson(), + e.fileItem.magicMetadata?.toEncodedJson(), + e.fileItem.pubMagicMetadata?.toEncodedJson(), + e.fileItem.info?.toEncodedJson(), + ], + ) + .toList(); + await _sqliteDB.executeBatch( + 'INSERT OR REPLACE INTO files ($filesColumns) values(?, ?, ?, ?, ?, ?, ?, ?)', + fileValues, + ); }); debugPrint( '$runtimeType insertCollectionFilesDiff complete in ${stopwatch.elapsed.inMilliseconds}ms for ${collections.length}', ); } - Future deleteCollectionFilesDiff( List items, ) async { diff --git a/mobile/lib/db/remote/migration.dart b/mobile/lib/db/remote/migration.dart index a59a40618c..1fb4b9fbba 100644 --- a/mobile/lib/db/remote/migration.dart +++ b/mobile/lib/db/remote/migration.dart @@ -11,9 +11,9 @@ const collectionFilesColumns = 'collection_id, file_id, enc_key, enc_key_nonce, created_at, updated_at, is_deleted'; const filesColumns = - 'id, owner_id, file_header, thumb_header, metadata, pri_medata, pub_medata, info'; + 'id, owner_id, file_header, thumb_header, metadata, priv_metadata, pub_metadata, info'; const trashedFilesColumns = - 'id, owner_id, file_header, thumb_header, metadata, pri_medata, pub_medata, info, trash_data'; + 'id, owner_id, file_header, thumb_header, metadata, priv_metadata, pub_metadata, info, trash_data'; String collectionValuePlaceHolder = collectionColumns.split(',').map((_) => '?').join(','); @@ -45,13 +45,13 @@ class RemoteDBMigration { CREATE TABLE collection_files ( file_id INTEGER NOT NULL, collection_id INTEGER NOT NULL, - PRIMARY KEY (file_id, collection_id) enc_key BLOB NOT NULL, enc_key_nonce BLOB NOT NULL, - is_deleted INTEGER NOT NULL + is_deleted INTEGER NOT NULL, updated_at INTEGER NOT NULL, created_at INTEGER NOT NULL DEFAULT 0, - ) + PRIMARY KEY (file_id, collection_id) + ); ''', ''' CREATE TABLE files ( @@ -59,12 +59,11 @@ class RemoteDBMigration { owner_id INTEGER NOT NULL, file_header BLOB NOT NULL, thumb_header BLOB NOT NULL, - metadata TEXT NOT NULL', - pri_medata TEXT NOT NULL DEFAULT '{}', - pub_medata TEXT NOT NULL DEFAULT '{}', + metadata TEXT NOT NULL, + priv_metadata TEXT NOT NULL DEFAULT '{}', + pub_metadata TEXT NOT NULL DEFAULT '{}', info TEXT DEFAULT '{}', - trash_data TEXT, - FOREIGN KEY(id) REFERENCES collection_files(file_id) + trash_data TEXT ) ''', ]; diff --git a/mobile/lib/models/api/diff/diff.dart b/mobile/lib/models/api/diff/diff.dart index 7f7d20a659..075f4315f9 100644 --- a/mobile/lib/models/api/diff/diff.dart +++ b/mobile/lib/models/api/diff/diff.dart @@ -1,24 +1,66 @@ +import "dart:convert"; import "dart:typed_data"; class Info { final int fileSize; - final int thumbnailSize; + final int thumbSize; static Info? fromJson(Map? json) { if (json == null) return null; return Info( - fileSize: json['fileSize'], - thumbnailSize: json['thumbnailSize'], + fileSize: json['fileSize'] ?? 0, + thumbSize: json['thumbSize'] ?? 0, ); } - Info({required this.fileSize, required this.thumbnailSize}); + Info({required this.fileSize, required this.thumbSize}); + + Map toJson() { + return { + 'fileSize': fileSize, + 'thumbSize': thumbSize, + }; + } + + String toEncodedJson() { + return jsonEncode(toJson()); + } + + static Info? fromEncodedJson(String? encodedJson) { + if (encodedJson == null) return null; + return Info.fromJson(jsonDecode(encodedJson)); + } } class Metadata { final Map data; final int version; + Metadata({required this.data, required this.version}); + + static fromJson(Map json) { + if (json.isEmpty || json['data'] == null) return null; + return Metadata( + data: json['data'], + version: json['version'], + ); + } + + Map toJson() { + return { + 'data': data, + 'version': version, + }; + } + + static Metadata? fromEncodedJson(String? encodedJson) { + if (encodedJson == null) return null; + return Metadata.fromJson(jsonDecode(encodedJson)); + } + + String toEncodedJson() { + return jsonEncode(toJson()); + } } class FileItem { diff --git a/mobile/lib/services/collections_service.dart b/mobile/lib/services/collections_service.dart index 559cc56a1f..cdf820c08b 100644 --- a/mobile/lib/services/collections_service.dart +++ b/mobile/lib/services/collections_service.dart @@ -48,8 +48,8 @@ import "package:photos/utils/local_settings.dart"; import 'package:shared_preferences/shared_preferences.dart'; class CollectionsService { - static const _collectionSyncTimeKeyPrefix = "collection_sync_time_x"; - static const _collectionsSyncTimeKey = "collections_sync_time_xx"; + static const _collectionSyncTimeKeyPrefix = "collection_sync_time_x3"; + static const _collectionsSyncTimeKey = "collections_sync_time_xx3"; static const int kMaximumWriteAttempts = 5; @@ -207,7 +207,9 @@ class CollectionsService { _cachedPublicAlbumKey.clear(); } - Future> getCollectionIDsToBeSynced() async { + Future> getCollectionIDsToBeSynced({ + bool newSync = false, + }) async { final idsToRemoveUpdateTimeMap = {}; for (final collection in _collectionIDToCollections.values) { if (!collection.isDeleted) { @@ -218,7 +220,8 @@ class CollectionsService { for (final MapEntry e in idsToRemoveUpdateTimeMap.entries) { final int cid = e.key; final int remoteUpdateTime = e.value; - if (remoteUpdateTime > getCollectionSyncTime(cid)) { + + if (remoteUpdateTime > getCollectionSyncTime(cid, syncV2: newSync)) { result[cid] = remoteUpdateTime; } } @@ -289,7 +292,12 @@ class CollectionsService { .toSet(); } - int getCollectionSyncTime(int collectionID) { + int getCollectionSyncTime(int collectionID, {bool syncV2 = false}) { + if (syncV2) { + return _prefs + .getInt('${_collectionSyncTimeKeyPrefix}_v2 $collectionID') ?? + 0; + } return _prefs .getInt(_collectionSyncTimeKeyPrefix + collectionID.toString()) ?? 0; diff --git a/mobile/lib/services/diff/remote_pull.dart b/mobile/lib/services/diff/remote_pull.dart index d049ea5357..fe17206560 100644 --- a/mobile/lib/services/diff/remote_pull.dart +++ b/mobile/lib/services/diff/remote_pull.dart @@ -6,14 +6,100 @@ import "dart:typed_data"; import "package:dio/dio.dart"; import "package:ente_crypto/ente_crypto.dart"; import "package:logging/logging.dart"; +import "package:photos/core/event_bus.dart"; +import "package:photos/events/diff_sync_complete_event.dart"; +import "package:photos/events/sync_status_update_event.dart"; import "package:photos/models/api/diff/diff.dart"; +import "package:photos/service_locator.dart"; import "package:photos/services/collections_service.dart"; import "package:photos/utils/file_uploader_util.dart"; class RemoteDiffService { final Logger _logger = Logger('RemoteDiffService'); final Dio _enteDio; - RemoteDiffService(this._enteDio); + final CollectionsService _collectionsService; + RemoteDiffService(this._enteDio, this._collectionsService); + + bool _isExistingSyncSilent = false; + + Future syncFromRemote() async { + _logger.info("Pulling remote diff"); + final isFirstSync = !_collectionsService.hasSyncedCollections(); + if (isFirstSync && !_isExistingSyncSilent) { + Bus.instance.fire(SyncStatusUpdate(SyncStatus.applyingRemoteDiff)); + } + await _collectionsService.sync(); + + final idsToRemoteUpdationTimeMap = + await _collectionsService.getCollectionIDsToBeSynced(); + await _syncUpdatedCollections(idsToRemoteUpdationTimeMap); + _isExistingSyncSilent = false; + // unawaited(_localFileUpdateService.markUpdatedFilesForReUpload()); + // unawaited(_notifyNewFiles + // + // (idsToRemoteUpdationTimeMap.keys.toList())); + } + + Future _syncUpdatedCollections( + final Map idsToRemoteUpdationTimeMap, + ) async { + for (final cid in idsToRemoteUpdationTimeMap.keys) { + await _syncCollectionDiff( + cid, + _collectionsService.getCollectionSyncTime(cid, syncV2: true), + ); + // update syncTime for the collection in sharedPrefs. Note: the + // syncTime can change on remote but we might not get a diff for the + // collection if there are not changes in the file, but the collection + // metadata (name, archive status, sharing etc) has changed. + final remoteUpdateTime = idsToRemoteUpdationTimeMap[cid]; + await _collectionsService.setCollectionSyncTime(cid, remoteUpdateTime); + } + _logger.info("All updated collections synced"); + Bus.instance.fire(DiffSyncCompleteEvent()); + } + + Future _syncCollectionDiff(int collectionID, int sinceTime) async { + _logger.info( + "[Collection-$collectionID] fetch diff silently: $_isExistingSyncSilent " + "since: $sinceTime", + ); + if (!_isExistingSyncSilent) { + Bus.instance.fire(SyncStatusUpdate(SyncStatus.applyingRemoteDiff)); + } + final diff = await getCollectionItemsDiff(collectionID, sinceTime); + await remoteDB.deleteCollectionFilesDiff(diff.deletedItems); + + if (diff.updatedItems.isNotEmpty) { + await remoteDB.insertCollectionFilesDiff(diff.updatedItems); + _logger.info( + "[Collection-$collectionID] Updated ${diff.updatedItems.length} files" + " from remote", + ); + + // Bus.instance.fire( + // CollectionUpdatedEvent( + // collectionID, + // diff.updatedFiles, + // "syncUpdateFromRemote", + // ), + // ); + } + + if (diff.maxUpdatedAtTime > 0) { + await _collectionsService.setCollectionSyncTime( + collectionID, + diff.maxUpdatedAtTime, + ); + } + if (diff.hasMore) { + return await _syncCollectionDiff( + collectionID, + _collectionsService.getCollectionSyncTime(collectionID), + ); + } + _logger.info("[Collection-$collectionID] synced"); + } Future getCollectionItemsDiff( int collectionID, @@ -63,7 +149,7 @@ class RemoteDiffService { final fileKey = CryptoUtil.decryptSync(encFileKey, collectionKey, encFileKeyNonce); - final encodedMetadata = CryptoUtil.decryptSync( + final encodedMetadata = await CryptoUtil.decryptChaCha( CryptoUtil.base642bin(item["metadata"]["encryptedData"]), fileKey, CryptoUtil.base642bin(item["metadata"]["decryptionHeader"]), @@ -87,7 +173,7 @@ class RemoteDiffService { final utfEncodedMmd = CryptoUtil.decryptChaChaSync( CryptoUtil.base642bin(item['magicMetadata']['data']), fileKey, - CryptoUtil.base642bin(item['magicMetadata']['header']), + CryptoUtil.base642bin(item['magicMetadata']['decryptionHeader']), ); privateMagicMetadata = Metadata( data: jsonDecode(utf8.decode(utfEncodedMmd)), diff --git a/mobile/lib/services/remote_sync_service.dart b/mobile/lib/services/remote_sync_service.dart index a549d026c0..db49d380c2 100644 --- a/mobile/lib/services/remote_sync_service.dart +++ b/mobile/lib/services/remote_sync_service.dart @@ -7,6 +7,7 @@ import 'package:logging/logging.dart'; import 'package:photos/core/configuration.dart'; import 'package:photos/core/errors.dart'; import 'package:photos/core/event_bus.dart'; +import "package:photos/core/network/network.dart"; import 'package:photos/db/device_files_db.dart'; import 'package:photos/db/file_updation_db.dart'; import 'package:photos/db/files_db.dart'; @@ -26,6 +27,7 @@ import "package:photos/service_locator.dart"; import 'package:photos/services/app_lifecycle_service.dart'; import 'package:photos/services/collections_service.dart'; import 'package:photos/services/diff/diff_fetcher.dart'; +import "package:photos/services/diff/remote_pull.dart"; import 'package:photos/services/ignored_files_service.dart'; import 'package:photos/services/local_file_update_service.dart'; import "package:photos/services/notification_service.dart"; @@ -49,6 +51,8 @@ class RemoteSyncService { Completer? _existingSync; bool _isExistingSyncSilent = false; + late RemoteDiffService newService; + static const kHasSyncedArchiveKey = "has_synced_archive"; /* This setting is used to maintain a list of local IDs for videos that the user has manually marked for upload, even if the global video upload setting is currently disabled. @@ -77,6 +81,8 @@ class RemoteSyncService { void init(SharedPreferences preferences) { _prefs = preferences; + newService = + RemoteDiffService(NetworkClient.instance.enteDio, _collectionsService); Bus.instance.on().listen((event) async { if (event.type == EventType.addedOrUpdated) { @@ -203,6 +209,7 @@ class RemoteSyncService { } Future _pullDiff() async { + await newService.syncFromRemote(); _logger.info("Pulling remote diff"); final isFirstSync = !_collectionsService.hasSyncedCollections(); if (isFirstSync && !_isExistingSyncSilent) { @@ -308,7 +315,6 @@ class RemoteSyncService { Future joinAndSyncCollection( BuildContext context, int collectionID, - ) async { await _collectionsService.joinPublicCollection(context, collectionID); await _collectionsService.sync();