diff --git a/auth/pubspec.yaml b/auth/pubspec.yaml index e7ab13d0be..74c48b251a 100644 --- a/auth/pubspec.yaml +++ b/auth/pubspec.yaml @@ -1,7 +1,7 @@ name: ente_auth description: ente two-factor authenticator -version: 4.4.0+440 +version: 4.4.1+441 publish_to: none environment: diff --git a/mobile/lib/db/files_db.dart b/mobile/lib/db/files_db.dart index beb48f451a..58b5e9b97b 100644 --- a/mobile/lib/db/files_db.dart +++ b/mobile/lib/db/files_db.dart @@ -960,25 +960,6 @@ class FilesDB with SqlDbBase { return result; } - Future> getLocalFilesBackedUpWithoutLocation(int userId) async { - final db = await instance.sqliteAsyncDB; - final rows = await db.getAll( - ''' - SELECT DISTINCT $columnLocalID FROM $filesTable - WHERE $columnOwnerID = ? AND $columnLocalID IS NOT NULL AND - ($columnUploadedFileID IS NOT NULL AND $columnUploadedFileID IS NOT -1) - AND ($columnLatitude IS NULL OR $columnLongitude IS NULL OR - $columnLatitude = 0.0 or $columnLongitude = 0.0) - ''', - [userId], - ); - final result = []; - for (final row in rows) { - result.add(row[columnLocalID] as String); - } - return result; - } - Future> getAllFilesAfterDate({ required FileType fileType, required DateTime beginDate, diff --git a/mobile/lib/models/file/file.dart b/mobile/lib/models/file/file.dart index 4d1094f832..af39c06922 100644 --- a/mobile/lib/models/file/file.dart +++ b/mobile/lib/models/file/file.dart @@ -67,7 +67,7 @@ class EnteFile { file.deviceFolder = pathName; file.location = Location(latitude: asset.latitude, longitude: asset.longitude); - file.fileType = fileTypeFromAsset(asset); + file.fileType = enteTypeFromAsset(asset); file.creationTime = parseFileCreationTime(asset); file.modificationTime = asset.modifiedDateTime.microsecondsSinceEpoch; file.fileSubType = asset.subtype; @@ -83,7 +83,7 @@ class EnteFile { file.deviceFolder = asset.relativePath; file.location = Location(latitude: asset.latitude, longitude: asset.longitude); - file.fileType = fileTypeFromAsset(asset); + file.fileType = enteTypeFromAsset(asset); file.creationTime = asset.createDateTime.microsecondsSinceEpoch; file.modificationTime = asset.modifiedDateTime.microsecondsSinceEpoch; file.fileSubType = asset.subtype; @@ -224,7 +224,16 @@ class EnteFile { } final metadata = {}; - metadata["localID"] = isSharedMediaToAppSandbox ? null : localID; + metadata["localID"] = asset?.id; + final String? hashValue = mediaUploadData.hashData?.fileHash; + if (hashValue != null) { + metadata["hash"] = hashValue; + } + if (asset != null) { + metadata["subType"] = asset.subtype; + } + metadata["version"] = kCurrentMetadataVersion; + metadata["title"] = title; metadata["deviceFolder"] = deviceFolder; metadata["creationTime"] = creationTime; @@ -236,19 +245,11 @@ class EnteFile { metadata["latitude"] = location!.latitude; metadata["longitude"] = location!.longitude; } - if (fileSubType != null) { - metadata["subType"] = fileSubType; - } + if (duration != null) { metadata["duration"] = duration; } - final String? hashValue = mediaUploadData.hashData?.fileHash; - if (hashValue != null) { - metadata["hash"] = hashValue; - } - if (metadataVersion != null) { - metadata["version"] = metadataVersion; - } + return metadata; } diff --git a/mobile/lib/models/file/file_type.dart b/mobile/lib/models/file/file_type.dart index 6b2623e348..ba6da2b00b 100644 --- a/mobile/lib/models/file/file_type.dart +++ b/mobile/lib/models/file/file_type.dart @@ -35,7 +35,7 @@ FileType getFileType(int fileType) { } } -FileType fileTypeFromAsset(AssetEntity asset) { +FileType enteTypeFromAsset(AssetEntity asset) { FileType type = FileType.image; switch (asset.type) { case AssetType.image: diff --git a/mobile/lib/services/local/asset_entity.service.dart b/mobile/lib/services/local/asset_entity.service.dart index ce5101902d..23523a73d6 100644 --- a/mobile/lib/services/local/asset_entity.service.dart +++ b/mobile/lib/services/local/asset_entity.service.dart @@ -4,6 +4,7 @@ import "dart:io"; import "package:logging/logging.dart"; import "package:photo_manager/photo_manager.dart"; import "package:photos/core/errors.dart"; +import "package:photos/models/file/file_type.dart"; class AssetEntityService { static final Logger _logger = Logger("AssetEntityService"); @@ -44,4 +45,30 @@ class AssetEntityService { } return sourceFile; } + +// check if the assetType is still the same. This can happen for livePhotos +// if the user turns off the video using native photos app + static void assetType(AssetEntity asset, FileType type) { + final assetType = enteTypeFromAsset(asset); + if (assetType == type) { + return; + } + if (Platform.isIOS || Platform.isMacOS) { + if (assetType == FileType.image && type == FileType.livePhoto) { + throw InvalidFileError( + 'id ${asset.id}', + InvalidReason.livePhotoToImageTypeChanged, + ); + } else if (assetType == FileType.livePhoto && type == FileType.image) { + throw InvalidFileError( + 'id ${asset.id}', + InvalidReason.imageToLivePhotoTypeChanged, + ); + } + } + throw InvalidFileError( + 'fileType mismatch for id ${asset.id} assetType $assetType fileType ${type.name}', + InvalidReason.unknown, + ); + } } diff --git a/mobile/lib/services/local_file_update_service.dart b/mobile/lib/services/local_file_update_service.dart index d55c97144c..32a26e1fdd 100644 --- a/mobile/lib/services/local_file_update_service.dart +++ b/mobile/lib/services/local_file_update_service.dart @@ -3,12 +3,10 @@ import 'dart:core'; import 'dart:io'; import 'package:logging/logging.dart'; -import "package:photo_manager/photo_manager.dart"; import "package:photos/core/configuration.dart"; import 'package:photos/core/errors.dart'; import 'package:photos/db/file_updation_db.dart'; import 'package:photos/db/files_db.dart'; -import "package:photos/extensions/list.dart"; import 'package:photos/models/file/file.dart'; import 'package:photos/models/file/file_type.dart'; import 'package:photos/utils/file_uploader_util.dart'; @@ -21,10 +19,6 @@ class LocalFileUpdateService { late FileUpdationDB _fileUpdationDB; late SharedPreferences _prefs; late Logger _logger; - final String _androidMissingGPSImportDone = - 'fm_android_missing_gps_import_done'; - final String _androidMissingGPSCheckDone = - 'fm_android_missing_gps_check_done'; final List _oldMigrationKeys = [ 'fm_badCreationTime', @@ -37,6 +31,7 @@ class LocalFileUpdateService { 'fm_import_ios_live_photo_size', 'fm_ios_live_photo_check', 'fm_import_ios_live_photo_check', + 'fm_android_missing_gps_check_done', ]; Completer? _existingMigration; @@ -62,9 +57,6 @@ class LocalFileUpdateService { try { await _markFilesWhichAreActuallyUpdated(); _cleanUpOlderMigration().ignore(); - if (Platform.isAndroid) { - await _androidMissingGPSCheck(); - } } catch (e, s) { _logger.severe('failed to perform migration', e, s); } finally { @@ -90,6 +82,7 @@ class LocalFileUpdateService { 'badLocationCord', 'livePhotoSize', 'livePhotoCheck', + 'androidMissingGPS', ]); for (var element in _oldMigrationKeys) { await _prefs.remove(element); @@ -216,131 +209,6 @@ class LocalFileUpdateService { ); } - //#region Android Missing GPS specific methods ### - - Future _androidMissingGPSCheck() async { - if (_prefs.containsKey(_androidMissingGPSCheckDone)) { - return; - } - await _importAndroidBadGPSCandidate(); - // singleRunLimit indicates number of files to check during single - // invocation of this method. The limit act as a crude way to limit the - // resource consumed by the method - const int singleRunLimit = 500; - final localIDsToProcess = - await _fileUpdationDB.getLocalIDsForPotentialReUpload( - singleRunLimit, - FileUpdationDB.androidMissingGPS, - ); - if (localIDsToProcess.isNotEmpty) { - final chunksOf50 = localIDsToProcess.chunks(50); - for (final chunk in chunksOf50) { - final sTime = DateTime.now().microsecondsSinceEpoch; - final List futures = []; - final chunkOf10 = chunk.chunks(10); - for (final smallChunk in chunkOf10) { - futures.add(_checkForMissingGPS(smallChunk)); - } - await Future.wait(futures); - final eTime = DateTime.now().microsecondsSinceEpoch; - final d = Duration(microseconds: eTime - sTime); - _logger.info( - 'Performed missing GPS Location check for ${chunk.length} files ' - 'completed in ${d.inSeconds.toString()} secs', - ); - } - } else { - _logger.info('Completed android missing GPS check'); - await _prefs.setBool(_androidMissingGPSCheckDone, true); - } - } - - Future _checkForMissingGPS(List localIDs) async { - try { - final List localFiles = - await FilesDB.instance.getLocalFiles(localIDs); - final ownerID = Configuration.instance.getUserID()!; - final Set localIDsWithFile = {}; - final Set reuploadCandidate = {}; - final Set processedIDs = {}; - for (EnteFile file in localFiles) { - if (file.localID == null) continue; - // ignore files that are not uploaded or have different owner - if (!file.isUploaded || file.ownerID! != ownerID) { - processedIDs.add(file.localID!); - } - if (file.hasLocation) { - processedIDs.add(file.localID!); - } - } - for (EnteFile enteFile in localFiles) { - try { - if (enteFile.localID == null || - processedIDs.contains(enteFile.localID!)) { - continue; - } - - final localID = enteFile.localID!; - localIDsWithFile.add(localID); - final AssetEntity? entity = await AssetEntity.fromId(localID); - if (entity == null) { - processedIDs.add(localID); - } else { - final latLng = await entity.latlngAsync(); - if ((latLng.longitude ?? 0) == 0 || (latLng.latitude ?? 0) == 0) { - processedIDs.add(localID); - } else { - reuploadCandidate.add(localID); - processedIDs.add(localID); - } - } - } catch (e, s) { - processedIDs.add(enteFile.localID!); - _logger.severe('lat/long check file ${enteFile.toString()}', e, s); - } - } - for (String id in localIDs) { - // if the file with given localID doesn't exist, consider it as done. - if (!localIDsWithFile.contains(id)) { - processedIDs.add(id); - } - } - await FileUpdationDB.instance.insertMultiple( - reuploadCandidate.toList(), - FileUpdationDB.modificationTimeUpdated, - ); - await FileUpdationDB.instance.deleteByLocalIDs( - processedIDs.toList(), - FileUpdationDB.androidMissingGPS, - ); - } catch (e, s) { - _logger.severe('error while checking missing GPS', e, s); - } - } - - Future _importAndroidBadGPSCandidate() async { - if (_prefs.containsKey(_androidMissingGPSImportDone)) { - return; - } - final sTime = DateTime.now().microsecondsSinceEpoch; - _logger.info('importing files without missing GPS'); - final int ownerID = Configuration.instance.getUserID()!; - final fileLocalIDs = - await FilesDB.instance.getLocalFilesBackedUpWithoutLocation(ownerID); - await _fileUpdationDB.insertMultiple( - fileLocalIDs, - FileUpdationDB.androidMissingGPS, - ); - final eTime = DateTime.now().microsecondsSinceEpoch; - final d = Duration(microseconds: eTime - sTime); - _logger.info( - 'importing completed, total files count ${fileLocalIDs.length} and took ${d.inSeconds.toString()} seconds', - ); - await _prefs.setBool(_androidMissingGPSImportDone, true); - } - - //#endregion Android Missing GPS specific methods ### - Future getUploadData(EnteFile file) async { final mediaUploadData = await getUploadDataFromEnteFile(file); // delete the file from app's internal cache if it was copied to app @@ -351,18 +219,4 @@ class LocalFileUpdateService { } return mediaUploadData; } - - Future<(MediaUploadData, int)> getUploadDataWithSizeSize( - EnteFile file, - ) async { - final mediaUploadData = await getUploadDataFromEnteFile(file); - final int size = await mediaUploadData.sourceFile!.length(); - // delete the file from app's internal cache if it was copied to app - // for upload. Shared Media should only be cleared when the upload - // succeeds. - if (Platform.isIOS && mediaUploadData.sourceFile != null) { - await mediaUploadData.sourceFile?.delete(); - } - return (mediaUploadData, size); - } } diff --git a/mobile/lib/utils/file_uploader.dart b/mobile/lib/utils/file_uploader.dart index 8157aa6e7a..47380bfc7e 100644 --- a/mobile/lib/utils/file_uploader.dart +++ b/mobile/lib/utils/file_uploader.dart @@ -706,6 +706,9 @@ class FileUploader { ); final metadata = await file.getMetadataForUpload(mediaUploadData, exifTime); + final Map pubMetadata = + _buildPublicMagicData(mediaUploadData, exifTime, file.rAsset); + MetadataRequest? pubMetadataRequest; final fileDecryptionHeader = CryptoUtil.bin2base64(fileEncryptResult.header!); @@ -721,6 +724,14 @@ class FileUploader { ); final metadataDecryptionHeader = CryptoUtil.bin2base64(encryptedMetadataResult.header!); + + if (pubMetadata.isNotEmpty) { + pubMetadataRequest = await getPubMetadataRequest( + file, + pubMetadata, + fileEncryptResult.key!, + ); + } if (SyncService.instance.shouldStopSync()) { throw SyncStopRequestedError(); } @@ -763,16 +774,7 @@ class FileUploader { fileEncryptResult.key!, CollectionsService.instance.getCollectionKey(collectionID), ); - final Map pubMetadata = - _buildPublicMagicData(mediaUploadData, exifTime, file.rAsset); - MetadataRequest? pubMetadataRequest; - if (pubMetadata.isNotEmpty) { - pubMetadataRequest = await getPubMetadataRequest( - file, - pubMetadata, - fileEncryptResult.key!, - ); - } + remoteFile = await _createFile( file, collectionID, diff --git a/mobile/lib/utils/file_uploader_util.dart b/mobile/lib/utils/file_uploader_util.dart index 91229be1d9..48da045101 100644 --- a/mobile/lib/utils/file_uploader_util.dart +++ b/mobile/lib/utils/file_uploader_util.dart @@ -28,7 +28,6 @@ import "package:photos/services/local/asset_entity.service.dart"; import "package:photos/services/local/local_import.dart"; import "package:photos/services/local/shared_assert.service.dart"; import "package:photos/utils/exif_util.dart"; -import 'package:photos/utils/file_util.dart'; import "package:photos/utils/standalone/decode_image.dart"; import "package:uuid/uuid.dart"; import 'package:video_thumbnail/video_thumbnail.dart'; @@ -80,7 +79,7 @@ Future getUploadDataFromEnteFile( bool parseExif = false, }) async { if (file.isSharedMediaToAppSandbox) { - return await _getMediaUploadDataFromAppCache(file, parseExif); + return await _getSharedMediaUploadData(file); } else { return await _getMediaUploadDataFromAssetFile(file, parseExif); } @@ -96,10 +95,8 @@ Future _getMediaUploadDataFromAssetFile( String? zipHash; String fileHash; Map? exifData; - - // The timeouts are to safeguard against https://github.com/CaiJingLong/flutter_photo_manager/issues/467 final asset = await AssetEntityService.fromIDWithRetry(file.lAsset!.id); - _assertFileType(asset, file); + AssetEntityService.assetType(asset, file.fileType); if (Platform.isIOS) { trackOriginFetchForUploadOrML.put(file.lAsset!.id, true); } @@ -208,33 +205,6 @@ Future zip({ ); } -// check if the assetType is still the same. This can happen for livePhotos -// if the user turns off the video using native photos app -void _assertFileType(AssetEntity asset, EnteFile file) { - final assetType = fileTypeFromAsset(asset); - if (assetType == file.fileType) { - return; - } - if (Platform.isIOS || Platform.isMacOS) { - if (assetType == FileType.image && file.fileType == FileType.livePhoto) { - throw InvalidFileError( - 'id ${asset.id}', - InvalidReason.livePhotoToImageTypeChanged, - ); - } else if (assetType == FileType.livePhoto && - file.fileType == FileType.image) { - throw InvalidFileError( - 'id ${asset.id}', - InvalidReason.imageToLivePhotoTypeChanged, - ); - } - } - throw InvalidFileError( - 'fileType mismatch for id ${asset.id} assetType $assetType fileType ${file.fileType}', - InvalidReason.unknown, - ); -} - Future _decorateEnteFileData( EnteFile file, AssetEntity asset, @@ -285,10 +255,7 @@ Future getPubMetadataRequest( ); } -Future _getMediaUploadDataFromAppCache( - EnteFile file, - bool parseExif, -) async { +Future _getSharedMediaUploadData(EnteFile file) async { final localPath = SharedAssetService.getPath(file.localID!); final sourceFile = File(localPath); if (!sourceFile.existsSync()) {