// RemotePullService is a service that pulls the latest changes from the sever. import "dart:convert"; import "dart:math"; import "dart:typed_data"; import "package:dio/dio.dart"; import "package:ente_crypto/ente_crypto.dart"; import "package:logging/logging.dart"; import "package:photos/core/event_bus.dart"; import "package:photos/events/diff_sync_complete_event.dart"; import "package:photos/events/sync_status_update_event.dart"; import "package:photos/models/api/diff/diff.dart"; import "package:photos/service_locator.dart"; import "package:photos/services/collections_service.dart"; import "package:photos/utils/file_uploader_util.dart"; class RemoteDiffService { final Logger _logger = Logger('RemoteDiffService'); final Dio _enteDio; final CollectionsService _collectionsService; RemoteDiffService(this._enteDio, this._collectionsService); bool _isExistingSyncSilent = false; Future syncFromRemote() async { _logger.info("Pulling remote diff"); final isFirstSync = !_collectionsService.hasSyncedCollections(); if (isFirstSync && !_isExistingSyncSilent) { Bus.instance.fire(SyncStatusUpdate(SyncStatus.applyingRemoteDiff)); } await _collectionsService.sync(); final idsToRemoteUpdationTimeMap = await _collectionsService.getCollectionIDsToBeSynced(); await _syncUpdatedCollections(idsToRemoteUpdationTimeMap); _isExistingSyncSilent = false; // unawaited(_localFileUpdateService.markUpdatedFilesForReUpload()); // unawaited(_notifyNewFiles // // (idsToRemoteUpdationTimeMap.keys.toList())); } Future _syncUpdatedCollections( final Map idsToRemoteUpdationTimeMap, ) async { for (final cid in idsToRemoteUpdationTimeMap.keys) { await _syncCollectionDiff( cid, _collectionsService.getCollectionSyncTime(cid, syncV2: true), ); // update syncTime for the collection in sharedPrefs. Note: the // syncTime can change on remote but we might not get a diff for the // collection if there are not changes in the file, but the collection // metadata (name, archive status, sharing etc) has changed. final remoteUpdateTime = idsToRemoteUpdationTimeMap[cid]; await _collectionsService.setCollectionSyncTime(cid, remoteUpdateTime); } _logger.info("All updated collections synced"); Bus.instance.fire(DiffSyncCompleteEvent()); } Future _syncCollectionDiff(int collectionID, int sinceTime) async { _logger.info( "[Collection-$collectionID] fetch diff silently: $_isExistingSyncSilent " "since: $sinceTime", ); if (!_isExistingSyncSilent) { Bus.instance.fire(SyncStatusUpdate(SyncStatus.applyingRemoteDiff)); } final diff = await getCollectionItemsDiff(collectionID, sinceTime); await remoteDB.deleteCollectionFilesDiff(diff.deletedItems); if (diff.updatedItems.isNotEmpty) { await remoteDB.insertCollectionFilesDiff(diff.updatedItems); _logger.info( "[Collection-$collectionID] Updated ${diff.updatedItems.length} files" " from remote", ); // Bus.instance.fire( // CollectionUpdatedEvent( // collectionID, // diff.updatedFiles, // "syncUpdateFromRemote", // ), // ); } if (diff.maxUpdatedAtTime > 0) { await _collectionsService.setCollectionSyncTime( collectionID, diff.maxUpdatedAtTime, ); } if (diff.hasMore) { return await _syncCollectionDiff( collectionID, _collectionsService.getCollectionSyncTime(collectionID), ); } _logger.info("[Collection-$collectionID] synced"); } Future getCollectionItemsDiff( int collectionID, int sinceTime, ) async { try { final response = await _enteDio.get( "/collections/v2/diff", queryParameters: { "collectionID": collectionID, "sinceTime": sinceTime, }, ); int latestUpdatedAtTime = 0; final diff = response.data["diff"] as List; final bool hasMore = response.data["hasMore"] as bool; final startTime = DateTime.now(); final deletedFiles = []; final updatedFiles = []; final Uint8List collectionKey = CollectionsService.instance.getCollectionKey(collectionID); for (final item in diff) { final int fileID = item["id"] as int; final int collectionID = item["collectionID"]; final int ownerID = item["ownerID"]; final int collectionUpdationTime = item["updationTime"]; final bool isCollectionItemDeleted = item["isDeleted"]; latestUpdatedAtTime = max(latestUpdatedAtTime, collectionUpdationTime); if (isCollectionItemDeleted) { final deletedItem = CollectionFileItem( collectionID: collectionID, updatedAt: collectionUpdationTime, isDeleted: true, createdAt: item["createdAt"] ?? DateTime.now().millisecondsSinceEpoch, fileItem: FileItem.deleted(fileID, ownerID), ); deletedFiles.add(deletedItem); continue; } final Uint8List encFileKey = CryptoUtil.base642bin(item["encryptedKey"]); final Uint8List encFileKeyNonce = CryptoUtil.base642bin(item["keyDecryptionNonce"]); final fileKey = CryptoUtil.decryptSync(encFileKey, collectionKey, encFileKeyNonce); final encodedMetadata = await CryptoUtil.decryptChaCha( CryptoUtil.base642bin(item["metadata"]["encryptedData"]), fileKey, CryptoUtil.base642bin(item["metadata"]["decryptionHeader"]), ); final Map defaultMetdata = jsonDecode(utf8.decode(encodedMetadata)); if (!defaultMetdata.containsKey('version')) { defaultMetdata['version'] = 0; } if (defaultMetdata['hash'] == null && defaultMetdata.containsKey('imageHash') && defaultMetdata.containsKey('videoHash')) { // old web version was putting live photo hash in dfferent fields defaultMetdata['hash'] = '${defaultMetdata['imageHash']}$kLivePhotoHashSeparator${defaultMetdata['videoHash']}'; } Metadata? pubMagicMetadata; Metadata? privateMagicMetadata; if (item['magicMetadata'] != null) { final utfEncodedMmd = CryptoUtil.decryptChaChaSync( CryptoUtil.base642bin(item['magicMetadata']['data']), fileKey, CryptoUtil.base642bin(item['magicMetadata']['decryptionHeader']), ); privateMagicMetadata = Metadata( data: jsonDecode(utf8.decode(utfEncodedMmd)), version: item['magicMetadata']['version'], ); } if (item['pubMagicMetadata'] != null) { final utfEncodedMmd = CryptoUtil.decryptChaChaSync( CryptoUtil.base642bin(item['pubMagicMetadata']['data']), fileKey, CryptoUtil.base642bin(item['pubMagicMetadata']['header']), ); pubMagicMetadata = Metadata( data: jsonDecode(utf8.decode(utfEncodedMmd)), version: item['pubMagicMetadata']['version'], ); } final String fileDecryptionHeader = item["file"]["decryptionHeader"]; final String thumbnailDecryptionHeader = item["thumbnail"]["decryptionHeader"]; final Info? info = Info.fromJson(item["info"]); final CollectionFileItem file = CollectionFileItem( collectionID: collectionID, updatedAt: collectionUpdationTime, encFileKey: encFileKey, encFileKeyNonce: encFileKeyNonce, isDeleted: false, createdAt: item["createdAt"], fileItem: FileItem( fileID: fileID, ownerID: ownerID, thumnailDecryptionHeader: CryptoUtil.base642bin(thumbnailDecryptionHeader), fileDecryotionHeader: CryptoUtil.base642bin(fileDecryptionHeader), metadata: Metadata(data: defaultMetdata, version: 0), magicMetadata: privateMagicMetadata, pubMagicMetadata: pubMagicMetadata, info: info, ), ); updatedFiles.add(file); } _logger.info('[Collection-$collectionID] parsed ${diff.length} ' 'diff items ( ${updatedFiles.length} updated) in ${DateTime.now().difference(startTime).inMilliseconds}ms'); return DiffResult( updatedFiles, deletedFiles, hasMore, latestUpdatedAtTime, ); } catch (e, s) { _logger.severe(e, s); rethrow; } } } class DiffResult { final List updatedItems; final List deletedItems; final bool hasMore; final int maxUpdatedAtTime; DiffResult( this.updatedItems, this.deletedItems, this.hasMore, this.maxUpdatedAtTime, ); }