From 9d601a6b4ec4450fe96df7583cd2fd5803986e30 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Fri, 28 Feb 2025 13:06:38 +0530 Subject: [PATCH] [mob] Clean up migration for livePhoto --- mobile/lib/db/file_updation_db.dart | 1 - .../services/local_file_update_service.dart | 188 +----------------- 2 files changed, 4 insertions(+), 185 deletions(-) diff --git a/mobile/lib/db/file_updation_db.dart b/mobile/lib/db/file_updation_db.dart index 3d781b7867..e4f79380a8 100644 --- a/mobile/lib/db/file_updation_db.dart +++ b/mobile/lib/db/file_updation_db.dart @@ -14,7 +14,6 @@ class FileUpdationDB { static const tableName = 're_upload_tracker'; static const columnLocalID = 'local_id'; static const columnReason = 'reason'; - static const livePhotoCheck = 'livePhotoCheck'; static const androidMissingGPS = 'androidMissingGPS'; static const modificationTimeUpdated = 'modificationTimeUpdated'; diff --git a/mobile/lib/services/local_file_update_service.dart b/mobile/lib/services/local_file_update_service.dart index ce5a9080af..0f9c48dcc0 100644 --- a/mobile/lib/services/local_file_update_service.dart +++ b/mobile/lib/services/local_file_update_service.dart @@ -9,11 +9,8 @@ import 'package:photos/core/errors.dart'; import 'package:photos/db/file_updation_db.dart'; import 'package:photos/db/files_db.dart'; import "package:photos/extensions/list.dart"; -import "package:photos/extensions/stop_watch.dart"; -import "package:photos/models/file/extensions/file_props.dart"; import 'package:photos/models/file/file.dart'; import 'package:photos/models/file/file_type.dart'; -import "package:photos/services/files_service.dart"; import 'package:photos/utils/file_uploader_util.dart'; import 'package:photos/utils/file_util.dart'; import 'package:shared_preferences/shared_preferences.dart'; @@ -24,13 +21,11 @@ class LocalFileUpdateService { late FileUpdationDB _fileUpdationDB; late SharedPreferences _prefs; late Logger _logger; - final String _iosLivePhotoSizeMigrationDone = 'fm_ios_live_photo_check'; - final String _doneLivePhotoImport = 'fm_import_ios_live_photo_check'; final String _androidMissingGPSImportDone = 'fm_android_missing_gps_import_done'; final String _androidMissingGPSCheckDone = 'fm_android_missing_gps_check_done'; - static int twoHundredKb = 200 * 1024; + final List _oldMigrationKeys = [ 'fm_badCreationTime', 'fm_badCreationTimeCompleted', @@ -40,6 +35,8 @@ class LocalFileUpdateService { 'fm_badLocationMigrationDone', 'fm_ios_live_photo_size', 'fm_import_ios_live_photo_size', + 'fm_ios_live_photo_check', + 'fm_import_ios_live_photo_check', ]; Completer? _existingMigration; @@ -65,9 +62,6 @@ class LocalFileUpdateService { try { await _markFilesWhichAreActuallyUpdated(); _cleanUpOlderMigration().ignore(); - if (!Platform.isAndroid) { - await _handleLivePhotosSizedCheck(); - } if (Platform.isAndroid) { await _androidMissingGPSCheck(); } @@ -95,6 +89,7 @@ class LocalFileUpdateService { 'missingLocationV2', 'badLocationCord', 'livePhotoSize', + 'livePhotoCheck', ]); for (var element in _oldMigrationKeys) { await _prefs.remove(element); @@ -217,181 +212,6 @@ class LocalFileUpdateService { ); } - Future checkLivePhoto(EnteFile file) async { - if (file.localID == null || - file.localID!.isEmpty || - !file.isUploaded || - file.fileType != FileType.livePhoto || - !file.isOwner) { - return; - } - if (_prefs.containsKey(_iosLivePhotoSizeMigrationDone)) { - return; - } - final hasEntry = await _fileUpdationDB.isExisting( - file.localID!, - FileUpdationDB.livePhotoCheck, - ); - if (hasEntry) { - _logger.info('eager checkLivePhoto ${file.tag}'); - await _checkLivePhotoWithLowOrUnknownSize([file.localID!]); - } - } - - Future _handleLivePhotosSizedCheck() async { - try { - if (_prefs.containsKey(_iosLivePhotoSizeMigrationDone)) { - return; - } - await _importLivePhotoReUploadCandidates(); - - // singleRunLimit indicates number of files to check during single - // invocation of this method. The limit act as a crude way to limit the - // resource consumed by the method - const int singleRunLimit = 500; - final localIDsToProcess = - await _fileUpdationDB.getLocalIDsForPotentialReUpload( - singleRunLimit, - FileUpdationDB.livePhotoCheck, - ); - if (localIDsToProcess.isNotEmpty) { - final chunksOf50 = localIDsToProcess.chunks(50); - for (final chunk in chunksOf50) { - final sTime = DateTime.now().microsecondsSinceEpoch; - final List futures = []; - final chunkOf10 = chunk.chunks(10); - for (final smallChunk in chunkOf10) { - futures.add(_checkLivePhotoWithLowOrUnknownSize(smallChunk)); - } - await Future.wait(futures); - final eTime = DateTime.now().microsecondsSinceEpoch; - final d = Duration(microseconds: eTime - sTime); - _logger.info( - 'Performed hashCheck for ${chunk.length} livePhoto files ' - 'completed in ${d.inSeconds.toString()} secs', - ); - } - } else { - await _prefs.setBool(_iosLivePhotoSizeMigrationDone, true); - } - } catch (e, s) { - _logger.severe('error while checking livePhotoSize check', e, s); - } - } - - Future _checkLivePhotoWithLowOrUnknownSize( - List localIDsToProcess, - ) async { - final int userID = Configuration.instance.getUserID()!; - final List result = - await FilesDB.instance.getLocalFiles(localIDsToProcess); - final List localFilesForUser = []; - final Set localIDsWithFile = {}; - final Set missingSizeIDs = {}; - final Set processedIDs = {}; - for (EnteFile file in result) { - if (file.ownerID == null || file.ownerID == userID) { - localFilesForUser.add(file); - localIDsWithFile.add(file.localID!); - if (file.isUploaded && file.fileSize == null) { - missingSizeIDs.add(file.uploadedFileID!); - } - if (file.isUploaded && file.updationTime == null) { - // file already queued for re-upload - processedIDs.add(file.localID!); - } - } - } - if (missingSizeIDs.isNotEmpty) { - await FilesService.instance.backFillSizes(missingSizeIDs.toList()); - _logger.info('sizes back fill for ${missingSizeIDs.length} files'); - // return early, let the check run in the next batch - return; - } - - // if a file for localID doesn't exist, then mark it as processed - // otherwise the app will be stuck in retrying same set of ids - - for (String localID in localIDsToProcess) { - if (!localIDsWithFile.contains(localID)) { - processedIDs.add(localID); - } - } - _logger.info(" check ${localIDsToProcess.length} files for livePhotoSize, " - "missing file cnt ${processedIDs.length}"); - - for (EnteFile file in localFilesForUser) { - if (file.fileSize == null) { - _logger.info('fileSize still null, skip this file'); - continue; - } else if (file.fileType != FileType.livePhoto) { - _logger.severe('fileType is not livePhoto, skip this file'); - processedIDs.add(file.localID!); - continue; - } - if (processedIDs.contains(file.localID)) { - continue; - } - try { - late MediaUploadData uploadData; - late int mediaUploadSize; - (uploadData, mediaUploadSize) = await getUploadDataWithSizeSize(file); - if ((file.fileSize! - mediaUploadSize).abs() > twoHundredKb) { - _logger.info( - 'Re-upload livePhoto localHash ${uploadData.hashData?.fileHash ?? "null"} & localSize: $mediaUploadSize' - ' and remoteHash ${file.hash ?? "null"} & removeSize: ${file.fileSize!}', - ); - await FilesDB.instance.markFilesForReUpload( - userID, - file.localID!, - file.title, - file.location, - file.creationTime!, - file.modificationTime!, - file.fileType, - ); - } - processedIDs.add(file.localID!); - } on InvalidFileError catch (e) { - if (e.reason == InvalidReason.livePhotoToImageTypeChanged || - e.reason == InvalidReason.imageToLivePhotoTypeChanged) { - // let existing file update check handle this case - _fileUpdationDB.insertMultiple( - [file.localID!], - FileUpdationDB.modificationTimeUpdated, - ).ignore(); - } else { - _logger.severe("livePhoto check failed: invalid file ${file.tag}", e); - } - processedIDs.add(file.localID!); - } catch (e) { - _logger.severe("livePhoto check failed", e); - } finally {} - } - _logger.info('completed check for ${localIDsToProcess.length} files'); - await _fileUpdationDB.deleteByLocalIDs( - processedIDs.toList(), - FileUpdationDB.livePhotoCheck, - ); - } - - Future _importLivePhotoReUploadCandidates() async { - if (_prefs.containsKey(_doneLivePhotoImport)) { - return; - } - _logger.info('_importLivePhotoReUploadCandidates'); - final EnteWatch watch = EnteWatch("_importLivePhotoReUploadCandidates"); - final int ownerID = Configuration.instance.getUserID()!; - final List localIDs = - await FilesDB.instance.getLivePhotosForUser(ownerID); - await _fileUpdationDB.insertMultiple( - localIDs, - FileUpdationDB.livePhotoCheck, - ); - watch.log("imported ${localIDs.length} files"); - await _prefs.setBool(_doneLivePhotoImport, true); - } - //#region Android Missing GPS specific methods ### Future _androidMissingGPSCheck() async {