From 9c9309cc0620dd5d2606c58145b6f40501a037fb Mon Sep 17 00:00:00 2001 From: Prateek Sunal Date: Sat, 6 Apr 2024 21:24:14 +0530 Subject: [PATCH] feat: add multipart upload support --- mobile/lib/core/constants.dart | 10 + mobile/lib/utils/file_uploader.dart | 13 +- mobile/lib/utils/multipart_upload_util.dart | 255 ++++++++++++++++++++ mobile/pubspec.lock | 2 +- mobile/pubspec.yaml | 1 + 5 files changed, 278 insertions(+), 3 deletions(-) create mode 100644 mobile/lib/utils/multipart_upload_util.dart diff --git a/mobile/lib/core/constants.dart b/mobile/lib/core/constants.dart index 6f8f191154..640af728b3 100644 --- a/mobile/lib/core/constants.dart +++ b/mobile/lib/core/constants.dart @@ -1,3 +1,5 @@ +import "package:photos/utils/crypto_util.dart"; + const int thumbnailSmallSize = 256; const int thumbnailQuality = 50; const int thumbnailLargeSize = 512; @@ -45,6 +47,14 @@ class FFDefault { static const bool enablePasskey = false; } +// this is the chunk size of the un-encrypted file which is read and encrypted before uploading it as a single part. +const multipartPartSize = 20 * 1024 * 1024; + +const fileReaderChunkSize = encryptionChunkSize; + +final fileChunksCombinedForAUploadPart = + (multipartPartSize / fileReaderChunkSize).floor(); + const kDefaultProductionEndpoint = 'https://api.ente.io'; const int intMaxValue = 9223372036854775807; diff --git a/mobile/lib/utils/file_uploader.dart b/mobile/lib/utils/file_uploader.dart index 7c898f985e..df3e54aeca 100644 --- a/mobile/lib/utils/file_uploader.dart +++ b/mobile/lib/utils/file_uploader.dart @@ -37,6 +37,7 @@ import 'package:photos/utils/crypto_util.dart'; import 'package:photos/utils/file_download_util.dart'; import 'package:photos/utils/file_uploader_util.dart'; import "package:photos/utils/file_util.dart"; +import "package:photos/utils/multipart_upload_util.dart"; import 'package:shared_preferences/shared_preferences.dart'; import 'package:tuple/tuple.dart'; import "package:uuid/uuid.dart"; @@ -492,9 +493,17 @@ class FileUploader { final String thumbnailObjectKey = await _putFile(thumbnailUploadURL, encryptedThumbnailFile); - final fileUploadURL = await _getUploadURL(); - final String fileObjectKey = await _putFile(fileUploadURL, encryptedFile); + final count = await calculatePartCount( + await encryptedFile.length(), + ); + final fileUploadURLs = await getMultipartUploadURLs(count); + final fileObjectKey = fileUploadURLs.objectKey; + + await putMultipartFile(fileUploadURLs, encryptedFile); + + // final fileUploadURL = await _getUploadURL(); + // fileObjectKey = await _putFile(fileUploadURL, encryptedFile); final metadata = await file.getMetadataForUpload(mediaUploadData); final encryptedMetadataResult = await CryptoUtil.encryptChaCha( utf8.encode(jsonEncode(metadata)) as Uint8List, diff --git a/mobile/lib/utils/multipart_upload_util.dart b/mobile/lib/utils/multipart_upload_util.dart new file mode 100644 index 0000000000..18c3f4795b --- /dev/null +++ b/mobile/lib/utils/multipart_upload_util.dart @@ -0,0 +1,255 @@ +// ignore_for_file: implementation_imports + +import "dart:io"; + +import "package:dio/dio.dart"; +import "package:logging/logging.dart"; +import "package:photos/core/constants.dart"; +import "package:photos/core/network/network.dart"; +import "package:xml/src/xml/entities/named_entities.dart"; +import "package:xml/xml.dart"; + +final _enteDio = NetworkClient.instance.enteDio; +final _dio = NetworkClient.instance.getDio(); + +class MultipartUploadURLs { + final String objectKey; + final List partsURLs; + final String completeURL; + + MultipartUploadURLs({ + required this.objectKey, + required this.partsURLs, + required this.completeURL, + }); + + factory MultipartUploadURLs.fromMap(Map map) { + return MultipartUploadURLs( + objectKey: map["urls"]["objectKey"], + partsURLs: (map["urls"]["partURLs"] as List).cast(), + completeURL: map["urls"]["completeURL"], + ); + } +} + +Future calculatePartCount(int fileSize) async { + final partCount = (fileSize / multipartPartSize).ceil(); + return partCount; +} + +Future getMultipartUploadURLs(int count) async { + try { + final response = await _enteDio.get( + "/files/multipart-upload-urls", + queryParameters: { + "count": count, + }, + ); + + return MultipartUploadURLs.fromMap(response.data); + } on Exception catch (e) { + Logger("MultipartUploadURL").severe(e); + rethrow; + } +} + +Future putMultipartFile( + MultipartUploadURLs urls, + File encryptedFile, +) async { + // upload individual parts and get their etags + final etags = await uploadParts(urls.partsURLs, encryptedFile); + + print(etags); + + // complete the multipart upload + await completeMultipartUpload(etags, urls.completeURL); +} + +Future> uploadParts( + List partsURLs, + File encryptedFile, +) async { + final partsLength = partsURLs.length; + final etags = {}; + + for (int i = 0; i < partsLength; i++) { + final partURL = partsURLs[i]; + final isLastPart = i == partsLength - 1; + final fileSize = isLastPart + ? encryptedFile.lengthSync() % multipartPartSize + : multipartPartSize; + + final response = await _dio.put( + partURL, + data: encryptedFile.openRead( + i * multipartPartSize, + isLastPart ? null : multipartPartSize, + ), + options: Options( + headers: { + Headers.contentLengthHeader: fileSize, + }, + ), + ); + + final eTag = response.headers.value("etag"); + + if (eTag?.isEmpty ?? true) { + throw Exception('ETAG_MISSING'); + } + + etags[i] = eTag!; + } + + return etags; +} + +Future completeMultipartUpload( + Map partEtags, + String completeURL, +) async { + final body = convertJs2Xml({ + 'CompleteMultipartUpload': partEtags.entries.toList(), + }); + + print(body); + + try { + await _dio.post( + completeURL, + data: body, + options: Options( + contentType: "text/xml", + ), + ); + } catch (e) { + Logger("MultipartUpload").severe(e); + rethrow; + } +} + +// for converting the response to xml +String convertJs2Xml(Map json) { + final builder = XmlBuilder(); + buildXml(builder, json); + return builder.buildDocument().toXmlString( + pretty: true, + indent: ' ', + entityMapping: defaultMyEntityMapping, + ); +} + +void buildXml(XmlBuilder builder, dynamic node) { + if (node is Map) { + node.forEach((key, value) { + builder.element(key, nest: () => buildXml(builder, value)); + }); + } else if (node is List) { + for (var item in node) { + buildXml(builder, item); + } + } else { + builder.element( + "Part", + nest: () { + builder.attribute( + "PartNumber", + (node as MapEntry).key + 1, + ); + print(node.value); + builder.attribute("ETag", node.value); + }, + ); + } +} + +XmlEntityMapping defaultMyEntityMapping = MyXmlDefaultEntityMapping.xml(); + +class MyXmlDefaultEntityMapping extends XmlDefaultEntityMapping { + MyXmlDefaultEntityMapping.xml() : this(xmlEntities); + MyXmlDefaultEntityMapping.html() : this(htmlEntities); + MyXmlDefaultEntityMapping.html5() : this(html5Entities); + MyXmlDefaultEntityMapping(super.entities); + + @override + String encodeText(String input) => + input.replaceAllMapped(_textPattern, _textReplace); + + @override + String encodeAttributeValue(String input, XmlAttributeType type) { + switch (type) { + case XmlAttributeType.SINGLE_QUOTE: + return input.replaceAllMapped( + _singeQuoteAttributePattern, + _singeQuoteAttributeReplace, + ); + case XmlAttributeType.DOUBLE_QUOTE: + return input.replaceAllMapped( + _doubleQuoteAttributePattern, + _doubleQuoteAttributeReplace, + ); + } + } +} + +final _textPattern = RegExp(r'[&<>' + _highlyDiscouragedCharClass + r']'); + +String _textReplace(Match match) { + final toEscape = match.group(0)!; + switch (toEscape) { + case '<': + return '<'; + case '&': + return '&'; + case '>': + return '>'; + default: + return _asNumericCharacterReferences(toEscape); + } +} + +final _singeQuoteAttributePattern = + RegExp(r"['&<>\n\r\t" + _highlyDiscouragedCharClass + r']'); + +String _singeQuoteAttributeReplace(Match match) { + final toEscape = match.group(0)!; + switch (toEscape) { + case "'": + return ''; + case '&': + return '&'; + case '<': + return '<'; + case '>': + return '>'; + default: + return _asNumericCharacterReferences(toEscape); + } +} + +final _doubleQuoteAttributePattern = + RegExp(r'["&<>\n\r\t' + _highlyDiscouragedCharClass + r']'); + +String _doubleQuoteAttributeReplace(Match match) { + final toEscape = match.group(0)!; + switch (toEscape) { + case '"': + return ''; + case '&': + return '&'; + case '<': + return '<'; + case '>': + return '>'; + default: + return _asNumericCharacterReferences(toEscape); + } +} + +const _highlyDiscouragedCharClass = + r'\u0001-\u0008\u000b\u000c\u000e-\u001f\u007f-\u0084\u0086-\u009f'; + +String _asNumericCharacterReferences(String toEscape) => toEscape.runes + .map((rune) => '&#x${rune.toRadixString(16).toUpperCase()};') + .join(); diff --git a/mobile/pubspec.lock b/mobile/pubspec.lock index a4d34d05a9..681cfc110f 100644 --- a/mobile/pubspec.lock +++ b/mobile/pubspec.lock @@ -2497,7 +2497,7 @@ packages: source: hosted version: "0.2.0+3" xml: - dependency: transitive + dependency: "direct main" description: name: xml sha256: "5bc72e1e45e941d825fd7468b9b4cc3b9327942649aeb6fc5cdbf135f0a86e84" diff --git a/mobile/pubspec.yaml b/mobile/pubspec.yaml index 133863b693..c10bd9d1ac 100644 --- a/mobile/pubspec.yaml +++ b/mobile/pubspec.yaml @@ -171,6 +171,7 @@ dependencies: wallpaper_manager_flutter: ^0.0.2 wechat_assets_picker: ^8.6.3 widgets_to_image: ^0.0.2 + xml: ^6.3.0 dependency_overrides: # current fork of tfite_flutter_helper depends on ffi: ^1.x.x