From 8fdc7dcd89f72d97740cfced7696afb07997c57d Mon Sep 17 00:00:00 2001 From: laurenspriem Date: Wed, 6 Nov 2024 16:04:29 +0530 Subject: [PATCH] [mob][photos] Decode only in rust --- .../face_ml/face_recognition_service.dart | 14 +-- mobile/lib/src/rust/api/image_processing.dart | 16 ++- mobile/lib/src/rust/frb_generated.dart | 70 ++++++++++-- mobile/lib/src/rust/frb_generated.io.dart | 53 +++++++-- mobile/lib/utils/image_ml_util.dart | 108 ++++++++++-------- mobile/lib/utils/ml_util.dart | 21 +++- mobile/rust/src/api/image_processing.rs | 16 ++- mobile/rust/src/frb_generated.rs | 93 +++++++++------ 8 files changed, 277 insertions(+), 114 deletions(-) diff --git a/mobile/lib/services/machine_learning/face_ml/face_recognition_service.dart b/mobile/lib/services/machine_learning/face_ml/face_recognition_service.dart index f788fbf62e..f9bc812ff9 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_recognition_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_recognition_service.dart @@ -1,12 +1,12 @@ import "dart:async" show unawaited; import "dart:typed_data" show Uint8List, Float32List; -import "dart:ui" show Image; import "package:logging/logging.dart"; import "package:photos/core/event_bus.dart"; import "package:photos/events/diff_sync_complete_event.dart"; import "package:photos/events/people_changed_event.dart"; import "package:photos/models/api/entity/type.dart"; +import "package:photos/models/ml/face/dimension.dart"; import "package:photos/service_locator.dart"; import "package:photos/services/machine_learning/face_ml/face_detection/detection.dart"; import "package:photos/services/machine_learning/face_ml/face_detection/face_detection_service.dart"; @@ -71,9 +71,9 @@ class FaceRecognitionService { static Future> runFacesPipeline( int enteFileID, - Image image, + Dimensions imageDimensions, Uint8List rawRgbaBytes, - Uint8List resizedBytes, + Uint8List resizedRgbBytes, int resizedHeight, int resizedWidth, int faceDetectionAddress, @@ -86,7 +86,7 @@ class FaceRecognitionService { final List faceDetectionResult = await _detectFacesSync( enteFileID, - resizedBytes, + resizedRgbBytes, resizedHeight, resizedWidth, faceDetectionAddress, @@ -105,7 +105,7 @@ class FaceRecognitionService { // Align the faces final Float32List faceAlignmentResult = await _alignFacesSync( - image, + imageDimensions, rawRgbaBytes, faceDetectionResult, faceResults, @@ -173,7 +173,7 @@ class FaceRecognitionService { /// Aligns multiple faces from the given image data. /// Returns a list of the aligned faces as image data. static Future _alignFacesSync( - Image image, + Dimensions imageDimensions, Uint8List rawRgbaBytes, List faces, List faceResults, @@ -181,7 +181,7 @@ class FaceRecognitionService { try { final (alignedFaces, alignmentResults, _, blurValues, _) = await preprocessToMobileFaceNetFloat32List( - image, + imageDimensions, rawRgbaBytes, faces, ); diff --git a/mobile/lib/src/rust/api/image_processing.dart b/mobile/lib/src/rust/api/image_processing.dart index 69a6d84fc3..6dbbb6d93c 100644 --- a/mobile/lib/src/rust/api/image_processing.dart +++ b/mobile/lib/src/rust/api/image_processing.dart @@ -8,8 +8,20 @@ import 'package:flutter_rust_bridge/flutter_rust_bridge_for_generated.dart'; import 'package:photos/src/rust/frb_generated.dart'; -Future<(Uint8List, BigInt, BigInt, Uint8List, BigInt, BigInt)> - processImageMlFromPath({required String imagePath}) => RustLib.instance.api +Future< + ( + Uint8List, + BigInt, + BigInt, + Uint8List, + BigInt, + BigInt, + Uint8List, + BigInt, + BigInt + )> processImageMlFromPath( + {required String imagePath}) => + RustLib.instance.api .crateApiImageProcessingProcessImageMlFromPath(imagePath: imagePath); Future<(Uint8List, String, BigInt, BigInt)> processYoloFace( diff --git a/mobile/lib/src/rust/frb_generated.dart b/mobile/lib/src/rust/frb_generated.dart index 50042b923c..103bbd5bc7 100644 --- a/mobile/lib/src/rust/frb_generated.dart +++ b/mobile/lib/src/rust/frb_generated.dart @@ -86,7 +86,18 @@ abstract class RustLibApi extends BaseApi { Future<(Uint8List, String, BigInt, BigInt)> crateApiImageProcessingProcessClip({required String imagePath}); - Future<(Uint8List, BigInt, BigInt, Uint8List, BigInt, BigInt)> + Future< + ( + Uint8List, + BigInt, + BigInt, + Uint8List, + BigInt, + BigInt, + Uint8List, + BigInt, + BigInt + )> crateApiImageProcessingProcessImageMlFromPath( {required String imagePath}); @@ -133,7 +144,18 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { ); @override - Future<(Uint8List, BigInt, BigInt, Uint8List, BigInt, BigInt)> + Future< + ( + Uint8List, + BigInt, + BigInt, + Uint8List, + BigInt, + BigInt, + Uint8List, + BigInt, + BigInt + )> crateApiImageProcessingProcessImageMlFromPath( {required String imagePath}) { return handler.executeNormal(NormalTask( @@ -145,7 +167,7 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { }, codec: DcoCodec( decodeSuccessData: - dco_decode_record_list_prim_u_8_strict_usize_usize_list_prim_u_8_strict_usize_usize, + dco_decode_record_list_prim_u_8_strict_usize_usize_list_prim_u_8_strict_usize_usize_list_prim_u_8_strict_usize_usize, decodeErrorData: null, ), constMeta: kCrateApiImageProcessingProcessImageMlFromPathConstMeta, @@ -264,13 +286,16 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { BigInt, Uint8List, BigInt, + BigInt, + Uint8List, + BigInt, BigInt - ) dco_decode_record_list_prim_u_8_strict_usize_usize_list_prim_u_8_strict_usize_usize( + ) dco_decode_record_list_prim_u_8_strict_usize_usize_list_prim_u_8_strict_usize_usize_list_prim_u_8_strict_usize_usize( dynamic raw) { // Codec=Dco (DartCObject based), see doc to use other codecs final arr = raw as List; - if (arr.length != 6) { - throw Exception('Expected 6 elements, got ${arr.length}'); + if (arr.length != 9) { + throw Exception('Expected 9 elements, got ${arr.length}'); } return ( dco_decode_list_prim_u_8_strict(arr[0]), @@ -279,6 +304,9 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { dco_decode_list_prim_u_8_strict(arr[3]), dco_decode_usize(arr[4]), dco_decode_usize(arr[5]), + dco_decode_list_prim_u_8_strict(arr[6]), + dco_decode_usize(arr[7]), + dco_decode_usize(arr[8]), ); } @@ -333,8 +361,11 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { BigInt, Uint8List, BigInt, + BigInt, + Uint8List, + BigInt, BigInt - ) sse_decode_record_list_prim_u_8_strict_usize_usize_list_prim_u_8_strict_usize_usize( + ) sse_decode_record_list_prim_u_8_strict_usize_usize_list_prim_u_8_strict_usize_usize_list_prim_u_8_strict_usize_usize( SseDeserializer deserializer) { // Codec=Sse (Serialization based), see doc to use other codecs final var_field0 = sse_decode_list_prim_u_8_strict(deserializer); @@ -343,13 +374,19 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { final var_field3 = sse_decode_list_prim_u_8_strict(deserializer); final var_field4 = sse_decode_usize(deserializer); final var_field5 = sse_decode_usize(deserializer); + final var_field6 = sse_decode_list_prim_u_8_strict(deserializer); + final var_field7 = sse_decode_usize(deserializer); + final var_field8 = sse_decode_usize(deserializer); return ( var_field0, var_field1, var_field2, var_field3, var_field4, - var_field5 + var_field5, + var_field6, + var_field7, + var_field8 ); } @@ -420,8 +457,18 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { @protected void - sse_encode_record_list_prim_u_8_strict_usize_usize_list_prim_u_8_strict_usize_usize( - (Uint8List, BigInt, BigInt, Uint8List, BigInt, BigInt) self, + sse_encode_record_list_prim_u_8_strict_usize_usize_list_prim_u_8_strict_usize_usize_list_prim_u_8_strict_usize_usize( + ( + Uint8List, + BigInt, + BigInt, + Uint8List, + BigInt, + BigInt, + Uint8List, + BigInt, + BigInt + ) self, SseSerializer serializer) { // Codec=Sse (Serialization based), see doc to use other codecs sse_encode_list_prim_u_8_strict(self.$1, serializer); @@ -430,6 +477,9 @@ class RustLibApiImpl extends RustLibApiImplPlatform implements RustLibApi { sse_encode_list_prim_u_8_strict(self.$4, serializer); sse_encode_usize(self.$5, serializer); sse_encode_usize(self.$6, serializer); + sse_encode_list_prim_u_8_strict(self.$7, serializer); + sse_encode_usize(self.$8, serializer); + sse_encode_usize(self.$9, serializer); } @protected diff --git a/mobile/lib/src/rust/frb_generated.io.dart b/mobile/lib/src/rust/frb_generated.io.dart index b06988bf2e..07b33596d1 100644 --- a/mobile/lib/src/rust/frb_generated.io.dart +++ b/mobile/lib/src/rust/frb_generated.io.dart @@ -38,8 +38,11 @@ abstract class RustLibApiImplPlatform extends BaseApiImpl { BigInt, Uint8List, BigInt, + BigInt, + Uint8List, + BigInt, BigInt - ) dco_decode_record_list_prim_u_8_strict_usize_usize_list_prim_u_8_strict_usize_usize( + ) dco_decode_record_list_prim_u_8_strict_usize_usize_list_prim_u_8_strict_usize_usize_list_prim_u_8_strict_usize_usize( dynamic raw); @protected @@ -69,8 +72,11 @@ abstract class RustLibApiImplPlatform extends BaseApiImpl { BigInt, Uint8List, BigInt, + BigInt, + Uint8List, + BigInt, BigInt - ) sse_decode_record_list_prim_u_8_strict_usize_usize_list_prim_u_8_strict_usize_usize( + ) sse_decode_record_list_prim_u_8_strict_usize_usize_list_prim_u_8_strict_usize_usize_list_prim_u_8_strict_usize_usize( SseDeserializer deserializer); @protected @@ -120,9 +126,19 @@ abstract class RustLibApiImplPlatform extends BaseApiImpl { } @protected - void cst_api_fill_to_wire_record_list_prim_u_8_strict_usize_usize_list_prim_u_8_strict_usize_usize( - (Uint8List, BigInt, BigInt, Uint8List, BigInt, BigInt) apiObj, - wire_cst_record_list_prim_u_8_strict_usize_usize_list_prim_u_8_strict_usize_usize + void cst_api_fill_to_wire_record_list_prim_u_8_strict_usize_usize_list_prim_u_8_strict_usize_usize_list_prim_u_8_strict_usize_usize( + ( + Uint8List, + BigInt, + BigInt, + Uint8List, + BigInt, + BigInt, + Uint8List, + BigInt, + BigInt + ) apiObj, + wire_cst_record_list_prim_u_8_strict_usize_usize_list_prim_u_8_strict_usize_usize_list_prim_u_8_strict_usize_usize wireObj) { wireObj.field0 = cst_encode_list_prim_u_8_strict(apiObj.$1); wireObj.field1 = cst_encode_usize(apiObj.$2); @@ -130,6 +146,9 @@ abstract class RustLibApiImplPlatform extends BaseApiImpl { wireObj.field3 = cst_encode_list_prim_u_8_strict(apiObj.$4); wireObj.field4 = cst_encode_usize(apiObj.$5); wireObj.field5 = cst_encode_usize(apiObj.$6); + wireObj.field6 = cst_encode_list_prim_u_8_strict(apiObj.$7); + wireObj.field7 = cst_encode_usize(apiObj.$8); + wireObj.field8 = cst_encode_usize(apiObj.$9); } @protected @@ -151,8 +170,18 @@ abstract class RustLibApiImplPlatform extends BaseApiImpl { @protected void - sse_encode_record_list_prim_u_8_strict_usize_usize_list_prim_u_8_strict_usize_usize( - (Uint8List, BigInt, BigInt, Uint8List, BigInt, BigInt) self, + sse_encode_record_list_prim_u_8_strict_usize_usize_list_prim_u_8_strict_usize_usize_list_prim_u_8_strict_usize_usize( + ( + Uint8List, + BigInt, + BigInt, + Uint8List, + BigInt, + BigInt, + Uint8List, + BigInt, + BigInt + ) self, SseSerializer serializer); @protected @@ -358,7 +387,7 @@ final class wire_cst_record_list_prim_u_8_strict_string_usize_usize external int field3; } -final class wire_cst_record_list_prim_u_8_strict_usize_usize_list_prim_u_8_strict_usize_usize +final class wire_cst_record_list_prim_u_8_strict_usize_usize_list_prim_u_8_strict_usize_usize_list_prim_u_8_strict_usize_usize extends ffi.Struct { external ffi.Pointer field0; @@ -375,4 +404,12 @@ final class wire_cst_record_list_prim_u_8_strict_usize_usize_list_prim_u_8_stric @ffi.UintPtr() external int field5; + + external ffi.Pointer field6; + + @ffi.UintPtr() + external int field7; + + @ffi.UintPtr() + external int field8; } diff --git a/mobile/lib/utils/image_ml_util.dart b/mobile/lib/utils/image_ml_util.dart index 324b45b49e..f0fa8ec95d 100644 --- a/mobile/lib/utils/image_ml_util.dart +++ b/mobile/lib/utils/image_ml_util.dart @@ -228,7 +228,7 @@ Future> generateFaceThumbnailsUsingCanvas( } Future<(Float32List, Dimensions)> preprocessImageYoloFace( - Image image, + Dimensions image, Uint8List rawRgbaBytes, ) async { const requiredWidth = 640; @@ -336,7 +336,7 @@ Future resizedToPreprocessedClip( } Future preprocessImageClip( - Image image, + Dimensions image, Uint8List rawRgbaBytes, ) async { const int requiredWidth = 256; @@ -375,20 +375,20 @@ Future preprocessImageClip( Future<(Float32List, List, List, List, Size)> preprocessToMobileFaceNetFloat32List( - Image image, + Dimensions imageDimensions, Uint8List rawRgbaBytes, List relativeFaces, { int width = 112, int height = 112, }) async { final Size originalSize = - Size(image.width.toDouble(), image.height.toDouble()); + Size(imageDimensions.width.toDouble(), imageDimensions.height.toDouble()); final List absoluteFaces = relativeToAbsoluteDetections( relativeDetections: relativeFaces, - imageWidth: image.width, - imageHeight: image.height, + imageWidth: imageDimensions.width, + imageHeight: imageDimensions.height, ); final alignedImagesFloat32List = @@ -412,7 +412,7 @@ Future<(Float32List, List, List, List, Size)> alignmentResults.add(alignmentResult); _warpAffineFloat32List( - image, + imageDimensions, rawRgbaBytes, alignmentResult.affineMatrix, alignedImagesFloat32List, @@ -446,7 +446,7 @@ Future<(Float32List, List, List, List, Size)> RGB _readPixelColor( int x, int y, - Image image, + Dimensions image, Uint8List rgbaBytes, ) { if (y < 0 || y >= image.height || x < 0 || x >= image.width) { @@ -474,7 +474,7 @@ RGB _readPixelColor( RGB _getPixelBlurred( int x, int y, - Image image, + Dimensions image, Uint8List rgbaBytes, ) { double r = 0, g = 0, b = 0; @@ -551,7 +551,7 @@ Future _cropImage( } void _warpAffineFloat32List( - Image inputImage, + Dimensions inputImageDimensions, Uint8List rawRgbaBytes, List> affineMatrix, Float32List outputList, @@ -606,8 +606,12 @@ void _warpAffineFloat32List( final num xOrigin = (xTrans - b00) * a00Prime + (yTrans - b10) * a01Prime; final num yOrigin = (xTrans - b00) * a10Prime + (yTrans - b10) * a11Prime; - final RGB pixel = - _getPixelBicubic(xOrigin, yOrigin, inputImage, rawRgbaBytes); + final RGB pixel = _getPixelBicubic( + xOrigin, + yOrigin, + inputImageDimensions, + rawRgbaBytes, + ); // Set the new pixel outputList[startIndex + 3 * (yTrans * width + xTrans)] = @@ -640,7 +644,7 @@ Future _cropAndEncodeCanvas( RGB _getPixelBilinear( num fx, num fy, - Image image, + Dimensions image, Uint8List rawRgbaBytes, { bool antiAlias = false, }) { @@ -659,7 +663,7 @@ RGB _getPixelBilinear( final dy1 = 1.0 - dy; // Get the original pixels (with gaussian blur if antialias) - final RGB Function(int, int, Image, Uint8List) readPixel = + final RGB Function(int, int, Dimensions, Uint8List) readPixel = antiAlias ? _getPixelBlurred : _readPixelColor; final RGB pixel1 = readPixel(x0, y0, image, rawRgbaBytes); final RGB pixel2 = readPixel(x1, y0, image, rawRgbaBytes); @@ -684,9 +688,14 @@ RGB _getPixelBilinear( } /// Get the pixel value using Bicubic Interpolation. Code taken mainly from https://github.com/brendan-duncan/image/blob/6e407612752ffdb90b28cd5863c7f65856349348/lib/src/image/image.dart#L697 -RGB _getPixelBicubic(num fx, num fy, Image image, Uint8List rawRgbaBytes) { - fx = fx.clamp(0, image.width - 1); - fy = fy.clamp(0, image.height - 1); +RGB _getPixelBicubic( + num fx, + num fy, + Dimensions imageDimensions, + Uint8List rawRgbaBytes, +) { + fx = fx.clamp(0, imageDimensions.width - 1); + fy = fy.clamp(0, imageDimensions.height - 1); final x = fx.toInt() - (fx >= 0.0 ? 0 : 1); final px = x - 1; @@ -705,62 +714,69 @@ RGB _getPixelBicubic(num fx, num fy, Image image, Uint8List rawRgbaBytes) { dx * dx * (2 * ipp - 5 * icp + 4 * inp - iap) + dx * dx * dx * (-ipp + 3 * icp - 3 * inp + iap)); - final icc = _readPixelColor(x, y, image, rawRgbaBytes); + final icc = _readPixelColor(x, y, imageDimensions, rawRgbaBytes); - final ipp = - px < 0 || py < 0 ? icc : _readPixelColor(px, py, image, rawRgbaBytes); - final icp = px < 0 ? icc : _readPixelColor(x, py, image, rawRgbaBytes); - final inp = py < 0 || nx >= image.width + final ipp = px < 0 || py < 0 ? icc - : _readPixelColor(nx, py, image, rawRgbaBytes); - final iap = ax >= image.width || py < 0 + : _readPixelColor(px, py, imageDimensions, rawRgbaBytes); + final icp = + px < 0 ? icc : _readPixelColor(x, py, imageDimensions, rawRgbaBytes); + final inp = py < 0 || nx >= imageDimensions.width ? icc - : _readPixelColor(ax, py, image, rawRgbaBytes); + : _readPixelColor(nx, py, imageDimensions, rawRgbaBytes); + final iap = ax >= imageDimensions.width || py < 0 + ? icc + : _readPixelColor(ax, py, imageDimensions, rawRgbaBytes); final ip0 = cubic(dx, ipp.$1, icp.$1, inp.$1, iap.$1); final ip1 = cubic(dx, ipp.$2, icp.$2, inp.$2, iap.$2); final ip2 = cubic(dx, ipp.$3, icp.$3, inp.$3, iap.$3); // final ip3 = cubic(dx, ipp.a, icp.a, inp.a, iap.a); - final ipc = px < 0 ? icc : _readPixelColor(px, y, image, rawRgbaBytes); - final inc = - nx >= image.width ? icc : _readPixelColor(nx, y, image, rawRgbaBytes); - final iac = - ax >= image.width ? icc : _readPixelColor(ax, y, image, rawRgbaBytes); + final ipc = + px < 0 ? icc : _readPixelColor(px, y, imageDimensions, rawRgbaBytes); + final inc = nx >= imageDimensions.width + ? icc + : _readPixelColor(nx, y, imageDimensions, rawRgbaBytes); + final iac = ax >= imageDimensions.width + ? icc + : _readPixelColor(ax, y, imageDimensions, rawRgbaBytes); final ic0 = cubic(dx, ipc.$1, icc.$1, inc.$1, iac.$1); final ic1 = cubic(dx, ipc.$2, icc.$2, inc.$2, iac.$2); final ic2 = cubic(dx, ipc.$3, icc.$3, inc.$3, iac.$3); // final ic3 = cubic(dx, ipc.a, icc.a, inc.a, iac.a); - final ipn = px < 0 || ny >= image.height + final ipn = px < 0 || ny >= imageDimensions.height ? icc - : _readPixelColor(px, ny, image, rawRgbaBytes); - final icn = - ny >= image.height ? icc : _readPixelColor(x, ny, image, rawRgbaBytes); - final inn = nx >= image.width || ny >= image.height + : _readPixelColor(px, ny, imageDimensions, rawRgbaBytes); + final icn = ny >= imageDimensions.height ? icc - : _readPixelColor(nx, ny, image, rawRgbaBytes); - final ian = ax >= image.width || ny >= image.height + : _readPixelColor(x, ny, imageDimensions, rawRgbaBytes); + final inn = nx >= imageDimensions.width || ny >= imageDimensions.height ? icc - : _readPixelColor(ax, ny, image, rawRgbaBytes); + : _readPixelColor(nx, ny, imageDimensions, rawRgbaBytes); + final ian = ax >= imageDimensions.width || ny >= imageDimensions.height + ? icc + : _readPixelColor(ax, ny, imageDimensions, rawRgbaBytes); final in0 = cubic(dx, ipn.$1, icn.$1, inn.$1, ian.$1); final in1 = cubic(dx, ipn.$2, icn.$2, inn.$2, ian.$2); final in2 = cubic(dx, ipn.$3, icn.$3, inn.$3, ian.$3); // final in3 = cubic(dx, ipn.a, icn.a, inn.a, ian.a); - final ipa = px < 0 || ay >= image.height + final ipa = px < 0 || ay >= imageDimensions.height ? icc - : _readPixelColor(px, ay, image, rawRgbaBytes); - final ica = - ay >= image.height ? icc : _readPixelColor(x, ay, image, rawRgbaBytes); - final ina = nx >= image.width || ay >= image.height + : _readPixelColor(px, ay, imageDimensions, rawRgbaBytes); + final ica = ay >= imageDimensions.height ? icc - : _readPixelColor(nx, ay, image, rawRgbaBytes); - final iaa = ax >= image.width || ay >= image.height + : _readPixelColor(x, ay, imageDimensions, rawRgbaBytes); + final ina = nx >= imageDimensions.width || ay >= imageDimensions.height ? icc - : _readPixelColor(ax, ay, image, rawRgbaBytes); + : _readPixelColor(nx, ay, imageDimensions, rawRgbaBytes); + final iaa = ax >= imageDimensions.width || ay >= imageDimensions.height + ? icc + : _readPixelColor(ax, ay, imageDimensions, rawRgbaBytes); final ia0 = cubic(dx, ipa.$1, ica.$1, ina.$1, iaa.$1); final ia1 = cubic(dx, ipa.$2, ica.$2, ina.$2, iaa.$2); diff --git a/mobile/lib/utils/ml_util.dart b/mobile/lib/utils/ml_util.dart index 783e049050..1fa7f2467f 100644 --- a/mobile/lib/utils/ml_util.dart +++ b/mobile/lib/utils/ml_util.dart @@ -400,17 +400,26 @@ Future analyzeImageStatic(Map args) async { // Decode the image once to use for both face detection and alignment final safePath = await safePathFromImagepath(imagePath); - final (image, rawRgbaBytes) = await decodeImageFromPath(imagePath); + // final (image, rawRgbaBytes) = await decodeImageFromPath(imagePath); + + final ( + rawRgbaBytes, + imageHeight, + imageWidth, + faceBytes, + faceHeight, + faceWidth, + clipBytes, + clipHeight, + clipWidth + ) = await processImageMlFromPath(imagePath: safePath); final decodeTime = DateTime.now(); final decodeMs = decodeTime.difference(startTime).inMilliseconds; - - final (faceBytes, faceHeight, faceWidth, clipBytes, clipHeight, clipWidth) = - await processImageMlFromPath(imagePath: safePath); _logger.info( 'ML processing in rust took ${DateTime.now().difference(decodeTime).inMilliseconds} ms', ); final decodedImageSize = - Dimensions(height: image.height, width: image.width); + Dimensions(height: imageHeight.toInt(), width: imageWidth.toInt()); final result = MLResult.fromEnteFileID(enteFileID); result.decodedImageSize = decodedImageSize; @@ -419,7 +428,7 @@ Future analyzeImageStatic(Map args) async { runFaces ? FaceRecognitionService.runFacesPipeline( enteFileID, - image, + decodedImageSize, rawRgbaBytes, faceBytes, faceHeight.toInt(), diff --git a/mobile/rust/src/api/image_processing.rs b/mobile/rust/src/api/image_processing.rs index 70fc3d83d4..955d8a4d11 100644 --- a/mobile/rust/src/api/image_processing.rs +++ b/mobile/rust/src/api/image_processing.rs @@ -3,7 +3,17 @@ use rgb::FromSlice; pub fn process_image_ml_from_path( image_path: &str, -) -> (Vec, usize, usize, Vec, usize, usize) { +) -> ( + Vec, + usize, + usize, + Vec, + usize, + usize, + Vec, + usize, + usize, +) { // Load the image (~200ms) let img = image::open(image_path).expect("Failed to open image"); @@ -29,6 +39,7 @@ pub fn process_image_ml_from_path( } // Convert image to RGB8 (~150ms) + let rgba_decoded = img.to_rgba8().to_vec(); let rgb_img = img.into_rgb8(); // Convert RGB8 to Vec (~30ms) @@ -80,6 +91,9 @@ pub fn process_image_ml_from_path( result_clip.push(pixel.b); } ( + rgba_decoded, + height, + width, result_face, new_height_face, new_width_face, diff --git a/mobile/rust/src/frb_generated.rs b/mobile/rust/src/frb_generated.rs index 3f335840ad..f02586ab34 100644 --- a/mobile/rust/src/frb_generated.rs +++ b/mobile/rust/src/frb_generated.rs @@ -197,7 +197,19 @@ impl SseDecode for (Vec, String, usize, usize) { } } -impl SseDecode for (Vec, usize, usize, Vec, usize, usize) { +impl SseDecode + for ( + Vec, + usize, + usize, + Vec, + usize, + usize, + Vec, + usize, + usize, + ) +{ // Codec=Sse (Serialization based), see doc to use other codecs fn sse_decode(deserializer: &mut flutter_rust_bridge::for_generated::SseDeserializer) -> Self { let mut var_field0 = >::sse_decode(deserializer); @@ -206,8 +218,12 @@ impl SseDecode for (Vec, usize, usize, Vec, usize, usize) { let mut var_field3 = >::sse_decode(deserializer); let mut var_field4 = ::sse_decode(deserializer); let mut var_field5 = ::sse_decode(deserializer); + let mut var_field6 = >::sse_decode(deserializer); + let mut var_field7 = ::sse_decode(deserializer); + let mut var_field8 = ::sse_decode(deserializer); return ( - var_field0, var_field1, var_field2, var_field3, var_field4, var_field5, + var_field0, var_field1, var_field2, var_field3, var_field4, var_field5, var_field6, + var_field7, var_field8, ); } } @@ -299,7 +315,19 @@ impl SseEncode for (Vec, String, usize, usize) { } } -impl SseEncode for (Vec, usize, usize, Vec, usize, usize) { +impl SseEncode + for ( + Vec, + usize, + usize, + Vec, + usize, + usize, + Vec, + usize, + usize, + ) +{ // Codec=Sse (Serialization based), see doc to use other codecs fn sse_encode(self, serializer: &mut flutter_rust_bridge::for_generated::SseSerializer) { >::sse_encode(self.0, serializer); @@ -308,6 +336,9 @@ impl SseEncode for (Vec, usize, usize, Vec, usize, usize) { >::sse_encode(self.3, serializer); ::sse_encode(self.4, serializer); ::sse_encode(self.5, serializer); + >::sse_encode(self.6, serializer); + ::sse_encode(self.7, serializer); + ::sse_encode(self.8, serializer); } } @@ -396,21 +427,12 @@ mod io { ) } } - impl CstDecode<(Vec, usize, usize, Vec, usize, usize)> - for wire_cst_record_list_prim_u_8_strict_usize_usize_list_prim_u_8_strict_usize_usize - { - // Codec=Cst (C-struct based), see doc to use other codecs - fn cst_decode(self) -> (Vec, usize, usize, Vec, usize, usize) { - ( - self.field0.cst_decode(), - self.field1.cst_decode(), - self.field2.cst_decode(), - self.field3.cst_decode(), - self.field4.cst_decode(), - self.field5.cst_decode(), - ) + impl CstDecode<(Vec,usize,usize,Vec,usize,usize,Vec,usize,usize,)> for wire_cst_record_list_prim_u_8_strict_usize_usize_list_prim_u_8_strict_usize_usize_list_prim_u_8_strict_usize_usize { + // Codec=Cst (C-struct based), see doc to use other codecs + fn cst_decode(self) -> (Vec,usize,usize,Vec,usize,usize,Vec,usize,usize,) { + (self.field0.cst_decode(),self.field1.cst_decode(),self.field2.cst_decode(),self.field3.cst_decode(),self.field4.cst_decode(),self.field5.cst_decode(),self.field6.cst_decode(),self.field7.cst_decode(),self.field8.cst_decode(),) + } } - } impl NewWithNullPtr for wire_cst_record_list_prim_u_8_strict_string_usize_usize { fn new_with_null_ptr() -> Self { Self { @@ -426,25 +448,24 @@ mod io { Self::new_with_null_ptr() } } - impl NewWithNullPtr - for wire_cst_record_list_prim_u_8_strict_usize_usize_list_prim_u_8_strict_usize_usize - { - fn new_with_null_ptr() -> Self { - Self { - field0: core::ptr::null_mut(), - field1: Default::default(), - field2: Default::default(), - field3: core::ptr::null_mut(), - field4: Default::default(), - field5: Default::default(), + impl NewWithNullPtr for wire_cst_record_list_prim_u_8_strict_usize_usize_list_prim_u_8_strict_usize_usize_list_prim_u_8_strict_usize_usize { + fn new_with_null_ptr() -> Self { + Self { field0: core::ptr::null_mut(), +field1: Default::default(), +field2: Default::default(), +field3: core::ptr::null_mut(), +field4: Default::default(), +field5: Default::default(), +field6: core::ptr::null_mut(), +field7: Default::default(), +field8: Default::default(), } } } - } - impl Default for wire_cst_record_list_prim_u_8_strict_usize_usize_list_prim_u_8_strict_usize_usize { - fn default() -> Self { - Self::new_with_null_ptr() + impl Default for wire_cst_record_list_prim_u_8_strict_usize_usize_list_prim_u_8_strict_usize_usize_list_prim_u_8_strict_usize_usize { + fn default() -> Self { + Self::new_with_null_ptr() + } } - } #[no_mangle] pub extern "C" fn frbgen_photos_wire__crate__api__image_processing__process_clip( @@ -509,13 +530,17 @@ mod io { } #[repr(C)] #[derive(Clone, Copy)] - pub struct wire_cst_record_list_prim_u_8_strict_usize_usize_list_prim_u_8_strict_usize_usize { + pub struct wire_cst_record_list_prim_u_8_strict_usize_usize_list_prim_u_8_strict_usize_usize_list_prim_u_8_strict_usize_usize + { field0: *mut wire_cst_list_prim_u_8_strict, field1: usize, field2: usize, field3: *mut wire_cst_list_prim_u_8_strict, field4: usize, field5: usize, + field6: *mut wire_cst_list_prim_u_8_strict, + field7: usize, + field8: usize, } } #[cfg(not(target_family = "wasm"))]