diff --git a/mobile/lib/services/local_sync_service.dart b/mobile/lib/services/local_sync_service.dart index 1915ac30c2..d8237dd92a 100644 --- a/mobile/lib/services/local_sync_service.dart +++ b/mobile/lib/services/local_sync_service.dart @@ -23,6 +23,7 @@ import "package:photos/utils/debouncer.dart"; import "package:photos/utils/photo_manager_util.dart"; import "package:photos/utils/sqlite_util.dart"; import 'package:shared_preferences/shared_preferences.dart'; +import 'package:synchronized/synchronized.dart'; import 'package:tuple/tuple.dart'; class LocalSyncService { @@ -31,6 +32,7 @@ class LocalSyncService { late SharedPreferences _prefs; Completer? _existingSync; late Debouncer _changeCallbackDebouncer; + final Lock _lock = Lock(); static const kDbUpdationTimeKey = "db_updation_time"; static const kHasCompletedFirstImportKey = "has_completed_firstImport"; @@ -77,50 +79,57 @@ class LocalSyncService { } _existingSync = Completer(); final int ownerID = Configuration.instance.getUserID()!; - final existingLocalFileIDs = await _db.getExistingLocalFileIDs(ownerID); - _logger.info("${existingLocalFileIDs.length} localIDs were discovered"); + + // We use a lock to prevent synchronisation to occur while it is downloading + // as this introduces wrong entry in FilesDB due to race condition + // This is a fix for https://github.com/ente-io/ente/issues/4296 + await _lock.synchronized(() async { + final existingLocalFileIDs = await _db.getExistingLocalFileIDs(ownerID); + _logger.info("${existingLocalFileIDs.length} localIDs were discovered"); - final syncStartTime = DateTime.now().microsecondsSinceEpoch; - final lastDBUpdationTime = _prefs.getInt(kDbUpdationTimeKey) ?? 0; - final startTime = DateTime.now().microsecondsSinceEpoch; - if (lastDBUpdationTime != 0) { - await _loadAndStoreDiff( - existingLocalFileIDs, - fromTime: lastDBUpdationTime, - toTime: syncStartTime, - ); - } else { - // Load from 0 - 01.01.2010 - Bus.instance.fire(SyncStatusUpdate(SyncStatus.startedFirstGalleryImport)); - var startTime = 0; - var toYear = 2010; - var toTime = DateTime(toYear).microsecondsSinceEpoch; - while (toTime < syncStartTime) { + final syncStartTime = DateTime.now().microsecondsSinceEpoch; + final lastDBUpdationTime = _prefs.getInt(kDbUpdationTimeKey) ?? 0; + final startTime = DateTime.now().microsecondsSinceEpoch; + if (lastDBUpdationTime != 0) { + await _loadAndStoreDiff( + existingLocalFileIDs, + fromTime: lastDBUpdationTime, + toTime: syncStartTime, + ); + } else { + // Load from 0 - 01.01.2010 + Bus.instance.fire(SyncStatusUpdate(SyncStatus.startedFirstGalleryImport)); + var startTime = 0; + var toYear = 2010; + var toTime = DateTime(toYear).microsecondsSinceEpoch; + while (toTime < syncStartTime) { + await _loadAndStoreDiff( + existingLocalFileIDs, + fromTime: startTime, + toTime: toTime, + ); + startTime = toTime; + toYear++; + toTime = DateTime(toYear).microsecondsSinceEpoch; + } await _loadAndStoreDiff( existingLocalFileIDs, fromTime: startTime, - toTime: toTime, + toTime: syncStartTime, ); - startTime = toTime; - toYear++; - toTime = DateTime(toYear).microsecondsSinceEpoch; } - await _loadAndStoreDiff( - existingLocalFileIDs, - fromTime: startTime, - toTime: syncStartTime, - ); - } - if (!hasCompletedFirstImport()) { - await _prefs.setBool(kHasCompletedFirstImportKey, true); - await _refreshDeviceFolderCountAndCover(isFirstSync: true); - _logger.fine("first gallery import finished"); - Bus.instance - .fire(SyncStatusUpdate(SyncStatus.completedFirstGalleryImport)); - } - final endTime = DateTime.now().microsecondsSinceEpoch; - final duration = Duration(microseconds: endTime - startTime); - _logger.info("Load took " + duration.inMilliseconds.toString() + "ms"); + if (!hasCompletedFirstImport()) { + await _prefs.setBool(kHasCompletedFirstImportKey, true); + await _refreshDeviceFolderCountAndCover(isFirstSync: true); + _logger.fine("first gallery import finished"); + Bus.instance + .fire(SyncStatusUpdate(SyncStatus.completedFirstGalleryImport)); + } + final endTime = DateTime.now().microsecondsSinceEpoch; + final duration = Duration(microseconds: endTime - startTime); + _logger.info("Load took " + duration.inMilliseconds.toString() + "ms"); + }); + _existingSync?.complete(); _existingSync = null; } @@ -240,6 +249,10 @@ class LocalSyncService { } } + Lock getLock() { + return _lock; + } + bool hasGrantedPermissions() { return _prefs.getBool(kHasGrantedPermissionsKey) ?? false; } diff --git a/mobile/lib/ui/tools/editor/video_editor_page.dart b/mobile/lib/ui/tools/editor/video_editor_page.dart index 392d93034c..d96f0008a7 100644 --- a/mobile/lib/ui/tools/editor/video_editor_page.dart +++ b/mobile/lib/ui/tools/editor/video_editor_page.dart @@ -299,6 +299,8 @@ class _VideoEditorPageState extends State { ); } catch (_) { await dialog.hide(); + } finally { + await PhotoManager.startChangeNotify(); } } } diff --git a/mobile/lib/ui/viewer/actions/file_selection_actions_widget.dart b/mobile/lib/ui/viewer/actions/file_selection_actions_widget.dart index 00c06bb2ef..16f23e3f73 100644 --- a/mobile/lib/ui/viewer/actions/file_selection_actions_widget.dart +++ b/mobile/lib/ui/viewer/actions/file_selection_actions_widget.dart @@ -822,17 +822,27 @@ class _FileSelectionActionsWidgetState } Future _download(List files) async { + final totalFiles = files.length; + int downloadedFiles = 0; + final dialog = createProgressDialog( context, - S.of(context).downloading, + S.of(context).downloading + " ($downloadedFiles/$totalFiles)", isDismissible: true, ); await dialog.show(); try { + final downloadQueue = DownloadQueue(maxConcurrent: 5); final futures = []; for (final file in files) { if (file.localID == null) { - futures.add(downloadToGallery(file)); + futures.add( + downloadQueue.add(() async { + await downloadToGallery(file); + downloadedFiles++; + dialog.update(message: S.of(context).downloading + " ($downloadedFiles/$totalFiles)"); + }), + ); } } await Future.wait(futures); diff --git a/mobile/lib/utils/file_download_util.dart b/mobile/lib/utils/file_download_util.dart index 1014005c94..bb5fbeb323 100644 --- a/mobile/lib/utils/file_download_util.dart +++ b/mobile/lib/utils/file_download_util.dart @@ -1,3 +1,5 @@ +import "dart:async"; +import "dart:collection"; import 'dart:io'; import 'package:dio/dio.dart'; @@ -190,44 +192,49 @@ Future downloadToGallery(EnteFile file) async { type == FileType.livePhoto && Platform.isAndroid; AssetEntity? savedAsset; final File? fileToSave = await getFile(file); - //Disabling notifications for assets changing to insert the file into - //files db before triggering a sync. - await PhotoManager.stopChangeNotify(); - if (type == FileType.image) { - savedAsset = await PhotoManager.editor - .saveImageWithPath(fileToSave!.path, title: file.title!); - } else if (type == FileType.video) { - savedAsset = - await PhotoManager.editor.saveVideo(fileToSave!, title: file.title!); - } else if (type == FileType.livePhoto) { - final File? liveVideoFile = - await getFileFromServer(file, liveVideo: true); - if (liveVideoFile == null) { - throw AssertionError("Live video can not be null"); + // We use a lock to prevent synchronisation to occur while it is downloading + // as this introduces wrong entry in FilesDB due to race condition + // This is a fix for https://github.com/ente-io/ente/issues/4296 + await LocalSyncService.instance.getLock().synchronized(() async { + //Disabling notifications for assets changing to insert the file into + //files db before triggering a sync. + await PhotoManager.stopChangeNotify(); + if (type == FileType.image) { + savedAsset = await PhotoManager.editor + .saveImageWithPath(fileToSave!.path, title: file.title!); + } else if (type == FileType.video) { + savedAsset = + await PhotoManager.editor.saveVideo(fileToSave!, title: file.title!); + } else if (type == FileType.livePhoto) { + final File? liveVideoFile = + await getFileFromServer(file, liveVideo: true); + if (liveVideoFile == null) { + throw AssertionError("Live video can not be null"); + } + if (downloadLivePhotoOnDroid) { + await _saveLivePhotoOnDroid(fileToSave!, liveVideoFile, file); + } else { + savedAsset = await PhotoManager.editor.darwin.saveLivePhoto( + imageFile: fileToSave!, + videoFile: liveVideoFile, + title: file.title!, + ); + } } - if (downloadLivePhotoOnDroid) { - await _saveLivePhotoOnDroid(fileToSave!, liveVideoFile, file); - } else { - savedAsset = await PhotoManager.editor.darwin.saveLivePhoto( - imageFile: fileToSave!, - videoFile: liveVideoFile, - title: file.title!, - ); - } - } - if (savedAsset != null) { - file.localID = savedAsset.id; - await FilesDB.instance.insert(file); - Bus.instance.fire( - LocalPhotosUpdatedEvent( - [file], - source: "download", - ), - ); - } else if (!downloadLivePhotoOnDroid && savedAsset == null) { - _logger.severe('Failed to save assert of type $type'); - } + if (savedAsset != null) { + file.localID = savedAsset!.id; + await FilesDB.instance.insert(file); + Bus.instance.fire( + LocalPhotosUpdatedEvent( + [file], + source: "download", + ), + ); + } else if (!downloadLivePhotoOnDroid && savedAsset == null) { + _logger.severe('Failed to save assert of type $type'); + } + }); } catch (e) { _logger.severe("Failed to save file", e); rethrow; @@ -273,3 +280,37 @@ Future _saveLivePhotoOnDroid( ); await IgnoredFilesService.instance.cacheAndInsert([ignoreVideoFile]); } + +class DownloadQueue { + final int maxConcurrent; + final Queue Function()> _queue = Queue(); + int _runningTasks = 0; + + DownloadQueue({this.maxConcurrent = 5}); + + Future add(Future Function() task) async { + final completer = Completer(); + _queue.add(() async { + try { + await task(); + completer.complete(); + } catch (e) { + completer.completeError(e); + } finally { + _runningTasks--; + _processQueue(); + } + return completer.future; + }); + _processQueue(); + return completer.future; + } + + void _processQueue() { + while (_runningTasks < maxConcurrent && _queue.isNotEmpty) { + final task = _queue.removeFirst(); + _runningTasks++; + task(); + } + } +} \ No newline at end of file