TerribleDev/ente
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[mob] Clean up migration for livePhoto

Browse Source
This commit is contained in:
Neeraj Gupta
2025-02-28 13:06:38 +05:30
parent f49fcd7984
commit 9d601a6b4e
2 changed files with 4 additions and 185 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
mobile/lib/db/file_updation_db.dart
View File

@@ -14,7 +14,6 @@ class FileUpdationDB {
static const tableName = 're_upload_tracker';
static const columnLocalID = 'local_id';
static const columnReason = 'reason';
static const livePhotoCheck = 'livePhotoCheck';
static const androidMissingGPS = 'androidMissingGPS';
static const modificationTimeUpdated = 'modificationTimeUpdated';

188
mobile/lib/services/local_file_update_service.dart
View File

@@ -9,11 +9,8 @@ import 'package:photos/core/errors.dart';
import 'package:photos/db/file_updation_db.dart';
import 'package:photos/db/files_db.dart';
import "package:photos/extensions/list.dart";
import "package:photos/extensions/stop_watch.dart";
import "package:photos/models/file/extensions/file_props.dart";
import 'package:photos/models/file/file.dart';
import 'package:photos/models/file/file_type.dart';
import "package:photos/services/files_service.dart";
import 'package:photos/utils/file_uploader_util.dart';
import 'package:photos/utils/file_util.dart';
import 'package:shared_preferences/shared_preferences.dart';
@@ -24,13 +21,11 @@ class LocalFileUpdateService {
late FileUpdationDB _fileUpdationDB;
late SharedPreferences _prefs;
late Logger _logger;
final String _iosLivePhotoSizeMigrationDone = 'fm_ios_live_photo_check';
final String _doneLivePhotoImport = 'fm_import_ios_live_photo_check';
final String _androidMissingGPSImportDone =
'fm_android_missing_gps_import_done';
final String _androidMissingGPSCheckDone =
'fm_android_missing_gps_check_done';
static int twoHundredKb = 200 * 1024;
final List<String> _oldMigrationKeys = [
'fm_badCreationTime',
'fm_badCreationTimeCompleted',
@@ -40,6 +35,8 @@ class LocalFileUpdateService {
'fm_badLocationMigrationDone',
'fm_ios_live_photo_size',
'fm_import_ios_live_photo_size',
'fm_ios_live_photo_check',
'fm_import_ios_live_photo_check',
];
Completer<void>? _existingMigration;
@@ -65,9 +62,6 @@ class LocalFileUpdateService {
try {
await _markFilesWhichAreActuallyUpdated();
_cleanUpOlderMigration().ignore();
if (!Platform.isAndroid) {
await _handleLivePhotosSizedCheck();
}
if (Platform.isAndroid) {
await _androidMissingGPSCheck();
}
@@ -95,6 +89,7 @@ class LocalFileUpdateService {
'missingLocationV2',
'badLocationCord',
'livePhotoSize',
'livePhotoCheck',
]);
for (var element in _oldMigrationKeys) {
await _prefs.remove(element);
@@ -217,181 +212,6 @@ class LocalFileUpdateService {
);
}
Future<void> checkLivePhoto(EnteFile file) async {
if (file.localID == null ||
file.localID!.isEmpty ||
!file.isUploaded ||
file.fileType != FileType.livePhoto ||
!file.isOwner) {
return;
}
if (_prefs.containsKey(_iosLivePhotoSizeMigrationDone)) {
return;
}
final hasEntry = await _fileUpdationDB.isExisting(
file.localID!,
FileUpdationDB.livePhotoCheck,
);
if (hasEntry) {
_logger.info('eager checkLivePhoto ${file.tag}');
await _checkLivePhotoWithLowOrUnknownSize([file.localID!]);
}
}
Future<void> _handleLivePhotosSizedCheck() async {
try {
if (_prefs.containsKey(_iosLivePhotoSizeMigrationDone)) {
return;
}
await _importLivePhotoReUploadCandidates();
// singleRunLimit indicates number of files to check during single
// invocation of this method. The limit act as a crude way to limit the
// resource consumed by the method
const int singleRunLimit = 500;
final localIDsToProcess =
await _fileUpdationDB.getLocalIDsForPotentialReUpload(
singleRunLimit,
FileUpdationDB.livePhotoCheck,
);
if (localIDsToProcess.isNotEmpty) {
final chunksOf50 = localIDsToProcess.chunks(50);
for (final chunk in chunksOf50) {
final sTime = DateTime.now().microsecondsSinceEpoch;
final List<Future> futures = [];
final chunkOf10 = chunk.chunks(10);
for (final smallChunk in chunkOf10) {
futures.add(_checkLivePhotoWithLowOrUnknownSize(smallChunk));
}
await Future.wait(futures);
final eTime = DateTime.now().microsecondsSinceEpoch;
final d = Duration(microseconds: eTime - sTime);
_logger.info(
'Performed hashCheck for ${chunk.length} livePhoto files '
'completed in ${d.inSeconds.toString()} secs',
);
}
} else {
await _prefs.setBool(_iosLivePhotoSizeMigrationDone, true);
}
} catch (e, s) {
_logger.severe('error while checking livePhotoSize check', e, s);
}
}
Future<void> _checkLivePhotoWithLowOrUnknownSize(
List<String> localIDsToProcess,
) async {
final int userID = Configuration.instance.getUserID()!;
final List<EnteFile> result =
await FilesDB.instance.getLocalFiles(localIDsToProcess);
final List<EnteFile> localFilesForUser = [];
final Set<String> localIDsWithFile = {};
final Set<int> missingSizeIDs = {};
final Set<String> processedIDs = {};
for (EnteFile file in result) {
if (file.ownerID == null || file.ownerID == userID) {
localFilesForUser.add(file);
localIDsWithFile.add(file.localID!);
if (file.isUploaded && file.fileSize == null) {
missingSizeIDs.add(file.uploadedFileID!);
}
if (file.isUploaded && file.updationTime == null) {
// file already queued for re-upload
processedIDs.add(file.localID!);
}
}
}
if (missingSizeIDs.isNotEmpty) {
await FilesService.instance.backFillSizes(missingSizeIDs.toList());
_logger.info('sizes back fill for ${missingSizeIDs.length} files');
// return early, let the check run in the next batch
return;
}
// if a file for localID doesn't exist, then mark it as processed
// otherwise the app will be stuck in retrying same set of ids
for (String localID in localIDsToProcess) {
if (!localIDsWithFile.contains(localID)) {
processedIDs.add(localID);
}
}
_logger.info(" check ${localIDsToProcess.length} files for livePhotoSize, "
"missing file cnt ${processedIDs.length}");
for (EnteFile file in localFilesForUser) {
if (file.fileSize == null) {
_logger.info('fileSize still null, skip this file');
continue;
} else if (file.fileType != FileType.livePhoto) {
_logger.severe('fileType is not livePhoto, skip this file');
processedIDs.add(file.localID!);
continue;
}
if (processedIDs.contains(file.localID)) {
continue;
}
try {
late MediaUploadData uploadData;
late int mediaUploadSize;
(uploadData, mediaUploadSize) = await getUploadDataWithSizeSize(file);
if ((file.fileSize! - mediaUploadSize).abs() > twoHundredKb) {
_logger.info(
'Re-upload livePhoto localHash ${uploadData.hashData?.fileHash ?? "null"} & localSize: $mediaUploadSize'
' and remoteHash ${file.hash ?? "null"} & removeSize: ${file.fileSize!}',
);
await FilesDB.instance.markFilesForReUpload(
userID,
file.localID!,
file.title,
file.location,
file.creationTime!,
file.modificationTime!,
file.fileType,
);
}
processedIDs.add(file.localID!);
} on InvalidFileError catch (e) {
if (e.reason == InvalidReason.livePhotoToImageTypeChanged ||
e.reason == InvalidReason.imageToLivePhotoTypeChanged) {
// let existing file update check handle this case
_fileUpdationDB.insertMultiple(
[file.localID!],
FileUpdationDB.modificationTimeUpdated,
).ignore();
} else {
_logger.severe("livePhoto check failed: invalid file ${file.tag}", e);
}
processedIDs.add(file.localID!);
} catch (e) {
_logger.severe("livePhoto check failed", e);
} finally {}
}
_logger.info('completed check for ${localIDsToProcess.length} files');
await _fileUpdationDB.deleteByLocalIDs(
processedIDs.toList(),
FileUpdationDB.livePhotoCheck,
);
}
Future<void> _importLivePhotoReUploadCandidates() async {
if (_prefs.containsKey(_doneLivePhotoImport)) {
return;
}
_logger.info('_importLivePhotoReUploadCandidates');
final EnteWatch watch = EnteWatch("_importLivePhotoReUploadCandidates");
final int ownerID = Configuration.instance.getUserID()!;
final List<String> localIDs =
await FilesDB.instance.getLivePhotosForUser(ownerID);
await _fileUpdationDB.insertMultiple(
localIDs,
FileUpdationDB.livePhotoCheck,
);
watch.log("imported ${localIDs.length} files");
await _prefs.setBool(_doneLivePhotoImport, true);
}
//#region Android Missing GPS specific methods ###
Future<void> _androidMissingGPSCheck() async {