TerribleDev/ente
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[server] Persist remote pull response

Browse Source
This commit is contained in:
Neeraj Gupta
2025-02-28 10:56:15 +05:30
parent 070ab80be9
commit cbff68bc42
6 changed files with 183 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
mobile/lib/db/remote/db.dart
View File

@@ -16,7 +16,7 @@ var devLog = log;
enum RemoteTable { collections, collection_files, files, entities }
class RemoteDB {
static const _databaseName = "remote.db";
static const _databaseName = "remotex2.db";
static const _batchInsertMaxCount = 1000;
late final SqliteDatabase _sqliteDB;
@@ -82,13 +82,30 @@ class RemoteDB {
'INSERT OR REPLACE INTO collection_files ($collectionFilesColumns) values(?, ?, ?, ?, ?, ?, ?)',
values,
);
final List<List<Object?>> fileValues = slice
.map(
(e) => [
e.fileItem.fileID,
e.fileItem.ownerID,
e.fileItem.fileDecryotionHeader,
e.fileItem.thumnailDecryptionHeader,
e.fileItem.metadata?.toEncodedJson(),
e.fileItem.magicMetadata?.toEncodedJson(),
e.fileItem.pubMagicMetadata?.toEncodedJson(),
e.fileItem.info?.toEncodedJson(),
],
)
.toList();
await _sqliteDB.executeBatch(
'INSERT OR REPLACE INTO files ($filesColumns) values(?, ?, ?, ?, ?, ?, ?, ?)',
fileValues,
);
});
debugPrint(
'$runtimeType insertCollectionFilesDiff complete in ${stopwatch.elapsed.inMilliseconds}ms for ${collections.length}',
);
}
Future<void> deleteCollectionFilesDiff(
List<CollectionFileItem> items,
) async {

19
mobile/lib/db/remote/migration.dart
View File

@@ -11,9 +11,9 @@ const collectionFilesColumns =
'collection_id, file_id, enc_key, enc_key_nonce, created_at, updated_at, is_deleted';
const filesColumns =
'id, owner_id, file_header, thumb_header, metadata, pri_medata, pub_medata, info';
'id, owner_id, file_header, thumb_header, metadata, priv_metadata, pub_metadata, info';
const trashedFilesColumns =
'id, owner_id, file_header, thumb_header, metadata, pri_medata, pub_medata, info, trash_data';
'id, owner_id, file_header, thumb_header, metadata, priv_metadata, pub_metadata, info, trash_data';
String collectionValuePlaceHolder =
collectionColumns.split(',').map((_) => '?').join(',');
@@ -45,13 +45,13 @@ class RemoteDBMigration {
CREATE TABLE collection_files (
file_id INTEGER NOT NULL,
collection_id INTEGER NOT NULL,
PRIMARY KEY (file_id, collection_id)
enc_key BLOB NOT NULL,
enc_key_nonce BLOB NOT NULL,
is_deleted INTEGER NOT NULL
is_deleted INTEGER NOT NULL,
updated_at INTEGER NOT NULL,
created_at INTEGER NOT NULL DEFAULT 0,
)
PRIMARY KEY (file_id, collection_id)
);
''',
'''
CREATE TABLE files (
@@ -59,12 +59,11 @@ class RemoteDBMigration {
owner_id INTEGER NOT NULL,
file_header BLOB NOT NULL,
thumb_header BLOB NOT NULL,
metadata TEXT NOT NULL',
pri_medata TEXT NOT NULL DEFAULT '{}',
pub_medata TEXT NOT NULL DEFAULT '{}',
metadata TEXT NOT NULL,
priv_metadata TEXT NOT NULL DEFAULT '{}',
pub_metadata TEXT NOT NULL DEFAULT '{}',
info TEXT DEFAULT '{}',
trash_data TEXT,
FOREIGN KEY(id) REFERENCES collection_files(file_id)
trash_data TEXT
)
''',
];

50
mobile/lib/models/api/diff/diff.dart
View File

@@ -1,24 +1,66 @@
import "dart:convert";
import "dart:typed_data";
class Info {
final int fileSize;
final int thumbnailSize;
final int thumbSize;
static Info? fromJson(Map<String, dynamic>? json) {
if (json == null) return null;
return Info(
fileSize: json['fileSize'],
thumbnailSize: json['thumbnailSize'],
fileSize: json['fileSize'] ?? 0,
thumbSize: json['thumbSize'] ?? 0,
);
}
Info({required this.fileSize, required this.thumbnailSize});
Info({required this.fileSize, required this.thumbSize});
Map<String, dynamic> toJson() {
return {
'fileSize': fileSize,
'thumbSize': thumbSize,
};
}
String toEncodedJson() {
return jsonEncode(toJson());
}
static Info? fromEncodedJson(String? encodedJson) {
if (encodedJson == null) return null;
return Info.fromJson(jsonDecode(encodedJson));
}
}
class Metadata {
final Map<String, dynamic> data;
final int version;
Metadata({required this.data, required this.version});
static fromJson(Map<String, dynamic> json) {
if (json.isEmpty || json['data'] == null) return null;
return Metadata(
data: json['data'],
version: json['version'],
);
}
Map<String, dynamic> toJson() {
return {
'data': data,
'version': version,
};
}
static Metadata? fromEncodedJson(String? encodedJson) {
if (encodedJson == null) return null;
return Metadata.fromJson(jsonDecode(encodedJson));
}
String toEncodedJson() {
return jsonEncode(toJson());
}
}
class FileItem {

18
mobile/lib/services/collections_service.dart
View File

@@ -48,8 +48,8 @@ import "package:photos/utils/local_settings.dart";
import 'package:shared_preferences/shared_preferences.dart';
class CollectionsService {
static const _collectionSyncTimeKeyPrefix = "collection_sync_time_x";
static const _collectionsSyncTimeKey = "collections_sync_time_xx";
static const _collectionSyncTimeKeyPrefix = "collection_sync_time_x3";
static const _collectionsSyncTimeKey = "collections_sync_time_xx3";
static const int kMaximumWriteAttempts = 5;
@@ -207,7 +207,9 @@ class CollectionsService {
_cachedPublicAlbumKey.clear();
}
Future<Map<int, int>> getCollectionIDsToBeSynced() async {
Future<Map<int, int>> getCollectionIDsToBeSynced({
bool newSync = false,
}) async {
final idsToRemoveUpdateTimeMap = <int, int>{};
for (final collection in _collectionIDToCollections.values) {
if (!collection.isDeleted) {
@@ -218,7 +220,8 @@ class CollectionsService {
for (final MapEntry<int, int> e in idsToRemoveUpdateTimeMap.entries) {
final int cid = e.key;
final int remoteUpdateTime = e.value;
if (remoteUpdateTime > getCollectionSyncTime(cid)) {
if (remoteUpdateTime > getCollectionSyncTime(cid, syncV2: newSync)) {
result[cid] = remoteUpdateTime;
}
}
@@ -289,7 +292,12 @@ class CollectionsService {
.toSet();
}
int getCollectionSyncTime(int collectionID) {
int getCollectionSyncTime(int collectionID, {bool syncV2 = false}) {
if (syncV2) {
return _prefs
.getInt('${_collectionSyncTimeKeyPrefix}_v2 $collectionID') ??
0;
}
return _prefs
.getInt(_collectionSyncTimeKeyPrefix + collectionID.toString()) ??
0;

92
mobile/lib/services/diff/remote_pull.dart
View File

@@ -6,14 +6,100 @@ import "dart:typed_data";
import "package:dio/dio.dart";
import "package:ente_crypto/ente_crypto.dart";
import "package:logging/logging.dart";
import "package:photos/core/event_bus.dart";
import "package:photos/events/diff_sync_complete_event.dart";
import "package:photos/events/sync_status_update_event.dart";
import "package:photos/models/api/diff/diff.dart";
import "package:photos/service_locator.dart";
import "package:photos/services/collections_service.dart";
import "package:photos/utils/file_uploader_util.dart";
class RemoteDiffService {
final Logger _logger = Logger('RemoteDiffService');
final Dio _enteDio;
RemoteDiffService(this._enteDio);
final CollectionsService _collectionsService;
RemoteDiffService(this._enteDio, this._collectionsService);
bool _isExistingSyncSilent = false;
Future<void> syncFromRemote() async {
_logger.info("Pulling remote diff");
final isFirstSync = !_collectionsService.hasSyncedCollections();
if (isFirstSync && !_isExistingSyncSilent) {
Bus.instance.fire(SyncStatusUpdate(SyncStatus.applyingRemoteDiff));
}
await _collectionsService.sync();
final idsToRemoteUpdationTimeMap =
await _collectionsService.getCollectionIDsToBeSynced();
await _syncUpdatedCollections(idsToRemoteUpdationTimeMap);
_isExistingSyncSilent = false;
// unawaited(_localFileUpdateService.markUpdatedFilesForReUpload());
// unawaited(_notifyNewFiles
//
// (idsToRemoteUpdationTimeMap.keys.toList()));
}
Future<void> _syncUpdatedCollections(
final Map<int, int> idsToRemoteUpdationTimeMap,
) async {
for (final cid in idsToRemoteUpdationTimeMap.keys) {
await _syncCollectionDiff(
cid,
_collectionsService.getCollectionSyncTime(cid, syncV2: true),
);
// update syncTime for the collection in sharedPrefs. Note: the
// syncTime can change on remote but we might not get a diff for the
// collection if there are not changes in the file, but the collection
// metadata (name, archive status, sharing etc) has changed.
final remoteUpdateTime = idsToRemoteUpdationTimeMap[cid];
await _collectionsService.setCollectionSyncTime(cid, remoteUpdateTime);
}
_logger.info("All updated collections synced");
Bus.instance.fire(DiffSyncCompleteEvent());
}
Future<void> _syncCollectionDiff(int collectionID, int sinceTime) async {
_logger.info(
"[Collection-$collectionID] fetch diff silently: $_isExistingSyncSilent "
"since: $sinceTime",
);
if (!_isExistingSyncSilent) {
Bus.instance.fire(SyncStatusUpdate(SyncStatus.applyingRemoteDiff));
}
final diff = await getCollectionItemsDiff(collectionID, sinceTime);
await remoteDB.deleteCollectionFilesDiff(diff.deletedItems);
if (diff.updatedItems.isNotEmpty) {
await remoteDB.insertCollectionFilesDiff(diff.updatedItems);
_logger.info(
"[Collection-$collectionID] Updated ${diff.updatedItems.length} files"
" from remote",
);
// Bus.instance.fire(
// CollectionUpdatedEvent(
// collectionID,
// diff.updatedFiles,
// "syncUpdateFromRemote",
// ),
// );
}
if (diff.maxUpdatedAtTime > 0) {
await _collectionsService.setCollectionSyncTime(
collectionID,
diff.maxUpdatedAtTime,
);
}
if (diff.hasMore) {
return await _syncCollectionDiff(
collectionID,
_collectionsService.getCollectionSyncTime(collectionID),
);
}
_logger.info("[Collection-$collectionID] synced");
}
Future<DiffResult> getCollectionItemsDiff(
int collectionID,
@@ -63,7 +149,7 @@ class RemoteDiffService {
final fileKey =
CryptoUtil.decryptSync(encFileKey, collectionKey, encFileKeyNonce);
final encodedMetadata = CryptoUtil.decryptSync(
final encodedMetadata = await CryptoUtil.decryptChaCha(
CryptoUtil.base642bin(item["metadata"]["encryptedData"]),
fileKey,
CryptoUtil.base642bin(item["metadata"]["decryptionHeader"]),
@@ -87,7 +173,7 @@ class RemoteDiffService {
final utfEncodedMmd = CryptoUtil.decryptChaChaSync(
CryptoUtil.base642bin(item['magicMetadata']['data']),
fileKey,
CryptoUtil.base642bin(item['magicMetadata']['header']),
CryptoUtil.base642bin(item['magicMetadata']['decryptionHeader']),
);
privateMagicMetadata = Metadata(
data: jsonDecode(utf8.decode(utfEncodedMmd)),

8
mobile/lib/services/remote_sync_service.dart
View File

@@ -7,6 +7,7 @@ import 'package:logging/logging.dart';
import 'package:photos/core/configuration.dart';
import 'package:photos/core/errors.dart';
import 'package:photos/core/event_bus.dart';
import "package:photos/core/network/network.dart";
import 'package:photos/db/device_files_db.dart';
import 'package:photos/db/file_updation_db.dart';
import 'package:photos/db/files_db.dart';
@@ -26,6 +27,7 @@ import "package:photos/service_locator.dart";
import 'package:photos/services/app_lifecycle_service.dart';
import 'package:photos/services/collections_service.dart';
import 'package:photos/services/diff/diff_fetcher.dart';
import "package:photos/services/diff/remote_pull.dart";
import 'package:photos/services/ignored_files_service.dart';
import 'package:photos/services/local_file_update_service.dart';
import "package:photos/services/notification_service.dart";
@@ -49,6 +51,8 @@ class RemoteSyncService {
Completer<void>? _existingSync;
bool _isExistingSyncSilent = false;
late RemoteDiffService newService;
static const kHasSyncedArchiveKey = "has_synced_archive";
/* This setting is used to maintain a list of local IDs for videos that the user has manually
marked for upload, even if the global video upload setting is currently disabled.
@@ -77,6 +81,8 @@ class RemoteSyncService {
void init(SharedPreferences preferences) {
_prefs = preferences;
newService =
RemoteDiffService(NetworkClient.instance.enteDio, _collectionsService);
Bus.instance.on<LocalPhotosUpdatedEvent>().listen((event) async {
if (event.type == EventType.addedOrUpdated) {
@@ -203,6 +209,7 @@ class RemoteSyncService {
}
Future<void> _pullDiff() async {
await newService.syncFromRemote();
_logger.info("Pulling remote diff");
final isFirstSync = !_collectionsService.hasSyncedCollections();
if (isFirstSync && !_isExistingSyncSilent) {
@@ -308,7 +315,6 @@ class RemoteSyncService {
Future<void> joinAndSyncCollection(
BuildContext context,
int collectionID,
) async {
await _collectionsService.joinPublicCollection(context, collectionID);
await _collectionsService.sync();