TerribleDev/ente
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into remote_db

Browse Source
This commit is contained in:
Neeraj Gupta
2025-06-30 14:35:46 +05:30
parent 91a10634cc 0529c8cf9b
commit dbbb80a817
8 changed files with 60 additions and 228 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
auth/pubspec.yaml
View File

@@ -1,7 +1,7 @@
name: ente_auth
description: ente two-factor authenticator
version: 4.4.0+440
version: 4.4.1+441
publish_to: none
environment:

19
mobile/lib/db/files_db.dart
View File

@@ -960,25 +960,6 @@ class FilesDB with SqlDbBase {
return result;
}
Future<List<String>> getLocalFilesBackedUpWithoutLocation(int userId) async {
final db = await instance.sqliteAsyncDB;
final rows = await db.getAll(
'''
SELECT DISTINCT $columnLocalID FROM $filesTable
WHERE $columnOwnerID = ? AND $columnLocalID IS NOT NULL AND
($columnUploadedFileID IS NOT NULL AND $columnUploadedFileID IS NOT -1)
AND ($columnLatitude IS NULL OR $columnLongitude IS NULL OR
$columnLatitude = 0.0 or $columnLongitude = 0.0)
''',
[userId],
);
final result = <String>[];
for (final row in rows) {
result.add(row[columnLocalID] as String);
}
return result;
}
Future<List<EnteFile>> getAllFilesAfterDate({
required FileType fileType,
required DateTime beginDate,

27
mobile/lib/models/file/file.dart
View File

@@ -67,7 +67,7 @@ class EnteFile {
file.deviceFolder = pathName;
file.location =
Location(latitude: asset.latitude, longitude: asset.longitude);
file.fileType = fileTypeFromAsset(asset);
file.fileType = enteTypeFromAsset(asset);
file.creationTime = parseFileCreationTime(asset);
file.modificationTime = asset.modifiedDateTime.microsecondsSinceEpoch;
file.fileSubType = asset.subtype;
@@ -83,7 +83,7 @@ class EnteFile {
file.deviceFolder = asset.relativePath;
file.location =
Location(latitude: asset.latitude, longitude: asset.longitude);
file.fileType = fileTypeFromAsset(asset);
file.fileType = enteTypeFromAsset(asset);
file.creationTime = asset.createDateTime.microsecondsSinceEpoch;
file.modificationTime = asset.modifiedDateTime.microsecondsSinceEpoch;
file.fileSubType = asset.subtype;
@@ -224,7 +224,16 @@ class EnteFile {
}
final metadata = <String, dynamic>{};
metadata["localID"] = isSharedMediaToAppSandbox ? null : localID;
metadata["localID"] = asset?.id;
final String? hashValue = mediaUploadData.hashData?.fileHash;
if (hashValue != null) {
metadata["hash"] = hashValue;
}
if (asset != null) {
metadata["subType"] = asset.subtype;
}
metadata["version"] = kCurrentMetadataVersion;
metadata["title"] = title;
metadata["deviceFolder"] = deviceFolder;
metadata["creationTime"] = creationTime;
@@ -236,19 +245,11 @@ class EnteFile {
metadata["latitude"] = location!.latitude;
metadata["longitude"] = location!.longitude;
}
if (fileSubType != null) {
metadata["subType"] = fileSubType;
}
if (duration != null) {
metadata["duration"] = duration;
}
final String? hashValue = mediaUploadData.hashData?.fileHash;
if (hashValue != null) {
metadata["hash"] = hashValue;
}
if (metadataVersion != null) {
metadata["version"] = metadataVersion;
}
return metadata;
}

2
mobile/lib/models/file/file_type.dart
View File

@@ -35,7 +35,7 @@ FileType getFileType(int fileType) {
}
}
FileType fileTypeFromAsset(AssetEntity asset) {
FileType enteTypeFromAsset(AssetEntity asset) {
FileType type = FileType.image;
switch (asset.type) {
case AssetType.image:

27
mobile/lib/services/local/asset_entity.service.dart
View File

@@ -4,6 +4,7 @@ import "dart:io";
import "package:logging/logging.dart";
import "package:photo_manager/photo_manager.dart";
import "package:photos/core/errors.dart";
import "package:photos/models/file/file_type.dart";
class AssetEntityService {
static final Logger _logger = Logger("AssetEntityService");
@@ -44,4 +45,30 @@ class AssetEntityService {
}
return sourceFile;
}
// check if the assetType is still the same. This can happen for livePhotos
// if the user turns off the video using native photos app
static void assetType(AssetEntity asset, FileType type) {
final assetType = enteTypeFromAsset(asset);
if (assetType == type) {
return;
}
if (Platform.isIOS || Platform.isMacOS) {
if (assetType == FileType.image && type == FileType.livePhoto) {
throw InvalidFileError(
'id ${asset.id}',
InvalidReason.livePhotoToImageTypeChanged,
);
} else if (assetType == FileType.livePhoto && type == FileType.image) {
throw InvalidFileError(
'id ${asset.id}',
InvalidReason.imageToLivePhotoTypeChanged,
);
}
}
throw InvalidFileError(
'fileType mismatch for id ${asset.id} assetType $assetType fileType ${type.name}',
InvalidReason.unknown,
);
}
}

150
mobile/lib/services/local_file_update_service.dart
View File

@@ -3,12 +3,10 @@ import 'dart:core';
import 'dart:io';
import 'package:logging/logging.dart';
import "package:photo_manager/photo_manager.dart";
import "package:photos/core/configuration.dart";
import 'package:photos/core/errors.dart';
import 'package:photos/db/file_updation_db.dart';
import 'package:photos/db/files_db.dart';
import "package:photos/extensions/list.dart";
import 'package:photos/models/file/file.dart';
import 'package:photos/models/file/file_type.dart';
import 'package:photos/utils/file_uploader_util.dart';
@@ -21,10 +19,6 @@ class LocalFileUpdateService {
late FileUpdationDB _fileUpdationDB;
late SharedPreferences _prefs;
late Logger _logger;
final String _androidMissingGPSImportDone =
'fm_android_missing_gps_import_done';
final String _androidMissingGPSCheckDone =
'fm_android_missing_gps_check_done';
final List<String> _oldMigrationKeys = [
'fm_badCreationTime',
@@ -37,6 +31,7 @@ class LocalFileUpdateService {
'fm_import_ios_live_photo_size',
'fm_ios_live_photo_check',
'fm_import_ios_live_photo_check',
'fm_android_missing_gps_check_done',
];
Completer<void>? _existingMigration;
@@ -62,9 +57,6 @@ class LocalFileUpdateService {
try {
await _markFilesWhichAreActuallyUpdated();
_cleanUpOlderMigration().ignore();
if (Platform.isAndroid) {
await _androidMissingGPSCheck();
}
} catch (e, s) {
_logger.severe('failed to perform migration', e, s);
} finally {
@@ -90,6 +82,7 @@ class LocalFileUpdateService {
'badLocationCord',
'livePhotoSize',
'livePhotoCheck',
'androidMissingGPS',
]);
for (var element in _oldMigrationKeys) {
await _prefs.remove(element);
@@ -216,131 +209,6 @@ class LocalFileUpdateService {
);
}
//#region Android Missing GPS specific methods ###
Future<void> _androidMissingGPSCheck() async {
if (_prefs.containsKey(_androidMissingGPSCheckDone)) {
return;
}
await _importAndroidBadGPSCandidate();
// singleRunLimit indicates number of files to check during single
// invocation of this method. The limit act as a crude way to limit the
// resource consumed by the method
const int singleRunLimit = 500;
final localIDsToProcess =
await _fileUpdationDB.getLocalIDsForPotentialReUpload(
singleRunLimit,
FileUpdationDB.androidMissingGPS,
);
if (localIDsToProcess.isNotEmpty) {
final chunksOf50 = localIDsToProcess.chunks(50);
for (final chunk in chunksOf50) {
final sTime = DateTime.now().microsecondsSinceEpoch;
final List<Future> futures = [];
final chunkOf10 = chunk.chunks(10);
for (final smallChunk in chunkOf10) {
futures.add(_checkForMissingGPS(smallChunk));
}
await Future.wait(futures);
final eTime = DateTime.now().microsecondsSinceEpoch;
final d = Duration(microseconds: eTime - sTime);
_logger.info(
'Performed missing GPS Location check for ${chunk.length} files '
'completed in ${d.inSeconds.toString()} secs',
);
}
} else {
_logger.info('Completed android missing GPS check');
await _prefs.setBool(_androidMissingGPSCheckDone, true);
}
}
Future<void> _checkForMissingGPS(List<String> localIDs) async {
try {
final List<EnteFile> localFiles =
await FilesDB.instance.getLocalFiles(localIDs);
final ownerID = Configuration.instance.getUserID()!;
final Set<String> localIDsWithFile = {};
final Set<String> reuploadCandidate = {};
final Set<String> processedIDs = {};
for (EnteFile file in localFiles) {
if (file.localID == null) continue;
// ignore files that are not uploaded or have different owner
if (!file.isUploaded || file.ownerID! != ownerID) {
processedIDs.add(file.localID!);
}
if (file.hasLocation) {
processedIDs.add(file.localID!);
}
}
for (EnteFile enteFile in localFiles) {
try {
if (enteFile.localID == null ||
processedIDs.contains(enteFile.localID!)) {
continue;
}
final localID = enteFile.localID!;
localIDsWithFile.add(localID);
final AssetEntity? entity = await AssetEntity.fromId(localID);
if (entity == null) {
processedIDs.add(localID);
} else {
final latLng = await entity.latlngAsync();
if ((latLng.longitude ?? 0) == 0 || (latLng.latitude ?? 0) == 0) {
processedIDs.add(localID);
} else {
reuploadCandidate.add(localID);
processedIDs.add(localID);
}
}
} catch (e, s) {
processedIDs.add(enteFile.localID!);
_logger.severe('lat/long check file ${enteFile.toString()}', e, s);
}
}
for (String id in localIDs) {
// if the file with given localID doesn't exist, consider it as done.
if (!localIDsWithFile.contains(id)) {
processedIDs.add(id);
}
}
await FileUpdationDB.instance.insertMultiple(
reuploadCandidate.toList(),
FileUpdationDB.modificationTimeUpdated,
);
await FileUpdationDB.instance.deleteByLocalIDs(
processedIDs.toList(),
FileUpdationDB.androidMissingGPS,
);
} catch (e, s) {
_logger.severe('error while checking missing GPS', e, s);
}
}
Future<void> _importAndroidBadGPSCandidate() async {
if (_prefs.containsKey(_androidMissingGPSImportDone)) {
return;
}
final sTime = DateTime.now().microsecondsSinceEpoch;
_logger.info('importing files without missing GPS');
final int ownerID = Configuration.instance.getUserID()!;
final fileLocalIDs =
await FilesDB.instance.getLocalFilesBackedUpWithoutLocation(ownerID);
await _fileUpdationDB.insertMultiple(
fileLocalIDs,
FileUpdationDB.androidMissingGPS,
);
final eTime = DateTime.now().microsecondsSinceEpoch;
final d = Duration(microseconds: eTime - sTime);
_logger.info(
'importing completed, total files count ${fileLocalIDs.length} and took ${d.inSeconds.toString()} seconds',
);
await _prefs.setBool(_androidMissingGPSImportDone, true);
}
//#endregion Android Missing GPS specific methods ###
Future<MediaUploadData> getUploadData(EnteFile file) async {
final mediaUploadData = await getUploadDataFromEnteFile(file);
// delete the file from app's internal cache if it was copied to app
@@ -351,18 +219,4 @@ class LocalFileUpdateService {
}
return mediaUploadData;
}
Future<(MediaUploadData, int)> getUploadDataWithSizeSize(
EnteFile file,
) async {
final mediaUploadData = await getUploadDataFromEnteFile(file);
final int size = await mediaUploadData.sourceFile!.length();
// delete the file from app's internal cache if it was copied to app
// for upload. Shared Media should only be cleared when the upload
// succeeds.
if (Platform.isIOS && mediaUploadData.sourceFile != null) {
await mediaUploadData.sourceFile?.delete();
}
return (mediaUploadData, size);
}
}

22
mobile/lib/utils/file_uploader.dart
View File

@@ -706,6 +706,9 @@ class FileUploader {
);
final metadata =
await file.getMetadataForUpload(mediaUploadData, exifTime);
final Map<String, dynamic> pubMetadata =
_buildPublicMagicData(mediaUploadData, exifTime, file.rAsset);
MetadataRequest? pubMetadataRequest;
final fileDecryptionHeader =
CryptoUtil.bin2base64(fileEncryptResult.header!);
@@ -721,6 +724,14 @@ class FileUploader {
);
final metadataDecryptionHeader =
CryptoUtil.bin2base64(encryptedMetadataResult.header!);
if (pubMetadata.isNotEmpty) {
pubMetadataRequest = await getPubMetadataRequest(
file,
pubMetadata,
fileEncryptResult.key!,
);
}
if (SyncService.instance.shouldStopSync()) {
throw SyncStopRequestedError();
}
@@ -763,16 +774,7 @@ class FileUploader {
fileEncryptResult.key!,
CollectionsService.instance.getCollectionKey(collectionID),
);
final Map<String, dynamic> pubMetadata =
_buildPublicMagicData(mediaUploadData, exifTime, file.rAsset);
MetadataRequest? pubMetadataRequest;
if (pubMetadata.isNotEmpty) {
pubMetadataRequest = await getPubMetadataRequest(
file,
pubMetadata,
fileEncryptResult.key!,
);
}
remoteFile = await _createFile(
file,
collectionID,

39
mobile/lib/utils/file_uploader_util.dart
View File

@@ -28,7 +28,6 @@ import "package:photos/services/local/asset_entity.service.dart";
import "package:photos/services/local/local_import.dart";
import "package:photos/services/local/shared_assert.service.dart";
import "package:photos/utils/exif_util.dart";
import 'package:photos/utils/file_util.dart';
import "package:photos/utils/standalone/decode_image.dart";
import "package:uuid/uuid.dart";
import 'package:video_thumbnail/video_thumbnail.dart';
@@ -80,7 +79,7 @@ Future<MediaUploadData> getUploadDataFromEnteFile(
bool parseExif = false,
}) async {
if (file.isSharedMediaToAppSandbox) {
return await _getMediaUploadDataFromAppCache(file, parseExif);
return await _getSharedMediaUploadData(file);
} else {
return await _getMediaUploadDataFromAssetFile(file, parseExif);
}
@@ -96,10 +95,8 @@ Future<MediaUploadData> _getMediaUploadDataFromAssetFile(
String? zipHash;
String fileHash;
Map<String, IfdTag>? exifData;
// The timeouts are to safeguard against https://github.com/CaiJingLong/flutter_photo_manager/issues/467
final asset = await AssetEntityService.fromIDWithRetry(file.lAsset!.id);
_assertFileType(asset, file);
AssetEntityService.assetType(asset, file.fileType);
if (Platform.isIOS) {
trackOriginFetchForUploadOrML.put(file.lAsset!.id, true);
}
@@ -208,33 +205,6 @@ Future<void> zip({
);
}
// check if the assetType is still the same. This can happen for livePhotos
// if the user turns off the video using native photos app
void _assertFileType(AssetEntity asset, EnteFile file) {
final assetType = fileTypeFromAsset(asset);
if (assetType == file.fileType) {
return;
}
if (Platform.isIOS || Platform.isMacOS) {
if (assetType == FileType.image && file.fileType == FileType.livePhoto) {
throw InvalidFileError(
'id ${asset.id}',
InvalidReason.livePhotoToImageTypeChanged,
);
} else if (assetType == FileType.livePhoto &&
file.fileType == FileType.image) {
throw InvalidFileError(
'id ${asset.id}',
InvalidReason.imageToLivePhotoTypeChanged,
);
}
}
throw InvalidFileError(
'fileType mismatch for id ${asset.id} assetType $assetType fileType ${file.fileType}',
InvalidReason.unknown,
);
}
Future<void> _decorateEnteFileData(
EnteFile file,
AssetEntity asset,
@@ -285,10 +255,7 @@ Future<MetadataRequest> getPubMetadataRequest(
);
}
Future<MediaUploadData> _getMediaUploadDataFromAppCache(
EnteFile file,
bool parseExif,
) async {
Future<MediaUploadData> _getSharedMediaUploadData(EnteFile file) async {
final localPath = SharedAssetService.getPath(file.localID!);
final sourceFile = File(localPath);
if (!sourceFile.existsSync()) {