diff --git a/auth/android/app/src/main/AndroidManifest.xml b/auth/android/app/src/main/AndroidManifest.xml index a7b34f1add..7c7a8ba5f0 100644 --- a/auth/android/app/src/main/AndroidManifest.xml +++ b/auth/android/app/src/main/AndroidManifest.xml @@ -1,7 +1,7 @@ es CFBundleName - auth + Auth CFBundlePackageType APPL CFBundleShortVersionString diff --git a/auth/lib/ui/code_widget.dart b/auth/lib/ui/code_widget.dart index 7a9eae46f9..a690d977e3 100644 --- a/auth/lib/ui/code_widget.dart +++ b/auth/lib/ui/code_widget.dart @@ -279,7 +279,7 @@ class _CodeWidgetState extends State { crossAxisAlignment: CrossAxisAlignment.start, mainAxisAlignment: MainAxisAlignment.spaceBetween, children: [ - if (widget.code.type == Type.totp) + if (widget.code.type.isTOTPCompatible) CodeTimerProgress( period: widget.code.period, ), diff --git a/auth/lib/utils/totp_util.dart b/auth/lib/utils/totp_util.dart index 0d6a8bd68f..d0076ed41f 100644 --- a/auth/lib/utils/totp_util.dart +++ b/auth/lib/utils/totp_util.dart @@ -1,8 +1,12 @@ import 'package:ente_auth/models/code.dart'; import 'package:flutter/foundation.dart'; import 'package:otp/otp.dart' as otp; +import 'package:steam_totp/steam_totp.dart'; String getOTP(Code code) { + if (code.issuer.toLowerCase() == 'steam') { + return _getSteamCode(code); + } if (code.type == Type.hotp) { return _getHOTPCode(code); } @@ -26,7 +30,18 @@ String _getHOTPCode(Code code) { ); } +String _getSteamCode(Code code, [bool isNext = false]) { + final SteamTOTP steamtotp = SteamTOTP(secret: code.secret); + + return steamtotp.generate( + DateTime.now().millisecondsSinceEpoch ~/ 1000 + (isNext ? code.period : 0), + ); +} + String getNextTotp(Code code) { + if (code.issuer.toLowerCase() == 'steam') { + return _getSteamCode(code, true); + } return otp.OTP.generateTOTPCodeString( getSanitizedSecret(code.secret), DateTime.now().millisecondsSinceEpoch + code.period * 1000, diff --git a/auth/macos/Runner/Assets.xcassets/AppIcon.appiconset/1024-mac.png b/auth/macos/Runner/Assets.xcassets/AppIcon.appiconset/1024-mac.png index bbf24e4364..840c8bfc3a 100644 Binary files a/auth/macos/Runner/Assets.xcassets/AppIcon.appiconset/1024-mac.png and b/auth/macos/Runner/Assets.xcassets/AppIcon.appiconset/1024-mac.png differ diff --git a/auth/macos/Runner/Assets.xcassets/AppIcon.appiconset/128-mac.png b/auth/macos/Runner/Assets.xcassets/AppIcon.appiconset/128-mac.png index 2a210095a5..331be75f38 100644 Binary files a/auth/macos/Runner/Assets.xcassets/AppIcon.appiconset/128-mac.png and b/auth/macos/Runner/Assets.xcassets/AppIcon.appiconset/128-mac.png differ diff --git a/auth/macos/Runner/Assets.xcassets/AppIcon.appiconset/16-mac.png b/auth/macos/Runner/Assets.xcassets/AppIcon.appiconset/16-mac.png index fb83d3abe9..b1d5492fbd 100644 Binary files a/auth/macos/Runner/Assets.xcassets/AppIcon.appiconset/16-mac.png and b/auth/macos/Runner/Assets.xcassets/AppIcon.appiconset/16-mac.png differ diff --git a/auth/macos/Runner/Assets.xcassets/AppIcon.appiconset/256-mac.png b/auth/macos/Runner/Assets.xcassets/AppIcon.appiconset/256-mac.png index f64b470b01..c4a7d049b6 100644 Binary files a/auth/macos/Runner/Assets.xcassets/AppIcon.appiconset/256-mac.png and b/auth/macos/Runner/Assets.xcassets/AppIcon.appiconset/256-mac.png differ diff --git a/auth/macos/Runner/Assets.xcassets/AppIcon.appiconset/32-mac.png b/auth/macos/Runner/Assets.xcassets/AppIcon.appiconset/32-mac.png index c72e503af5..2cd08ec25c 100644 Binary files a/auth/macos/Runner/Assets.xcassets/AppIcon.appiconset/32-mac.png and b/auth/macos/Runner/Assets.xcassets/AppIcon.appiconset/32-mac.png differ diff --git a/auth/macos/Runner/Assets.xcassets/AppIcon.appiconset/512-mac.png b/auth/macos/Runner/Assets.xcassets/AppIcon.appiconset/512-mac.png index 07f8c930f9..53ff9127dd 100644 Binary files a/auth/macos/Runner/Assets.xcassets/AppIcon.appiconset/512-mac.png and b/auth/macos/Runner/Assets.xcassets/AppIcon.appiconset/512-mac.png differ diff --git a/auth/macos/Runner/Assets.xcassets/AppIcon.appiconset/64-mac.png b/auth/macos/Runner/Assets.xcassets/AppIcon.appiconset/64-mac.png index d7c149e3d1..daecae1f30 100644 Binary files a/auth/macos/Runner/Assets.xcassets/AppIcon.appiconset/64-mac.png and b/auth/macos/Runner/Assets.xcassets/AppIcon.appiconset/64-mac.png differ diff --git a/auth/pubspec.lock b/auth/pubspec.lock index a47858d533..b3a643b0be 100644 --- a/auth/pubspec.lock +++ b/auth/pubspec.lock @@ -745,6 +745,22 @@ packages: url: "https://pub.dev" source: hosted version: "2.1.0" + hashlib: + dependency: transitive + description: + name: hashlib + sha256: "67e640e19cc33070113acab3125cd48ebe480a0300e15554dec089b8878a729f" + url: "https://pub.dev" + source: hosted + version: "1.16.0" + hashlib_codecs: + dependency: transitive + description: + name: hashlib_codecs + sha256: "49e2a471f74b15f1854263e58c2ac11f2b631b5b12c836f9708a35397d36d626" + url: "https://pub.dev" + source: hosted + version: "2.2.0" hex: dependency: transitive description: @@ -1439,6 +1455,14 @@ packages: url: "https://pub.dev" source: hosted version: "1.11.1" + steam_totp: + dependency: "direct main" + description: + name: steam_totp + sha256: "3c09143c983f6bb05bb53e9232f9d40bbcc01c596ba0273c3e6bb246729abfa1" + url: "https://pub.dev" + source: hosted + version: "0.0.1" step_progress_indicator: dependency: "direct main" description: diff --git a/auth/pubspec.yaml b/auth/pubspec.yaml index 1488008138..0941b60711 100644 --- a/auth/pubspec.yaml +++ b/auth/pubspec.yaml @@ -1,6 +1,6 @@ name: ente_auth description: ente two-factor authenticator -version: 3.0.1+301 +version: 3.0.3+303 publish_to: none environment: @@ -94,6 +94,7 @@ dependencies: sqflite_common_ffi: ^2.3.0+4 sqlite3: ^2.1.0 sqlite3_flutter_libs: ^0.5.19+1 + steam_totp: ^0.0.1 step_progress_indicator: ^1.0.2 styled_text: ^8.1.0 tray_manager: ^0.2.1 diff --git a/desktop/src/main/services/dir.ts b/desktop/src/main/services/dir.ts index 9473be7c9e..4b1f748fe5 100644 --- a/desktop/src/main/services/dir.ts +++ b/desktop/src/main/services/dir.ts @@ -51,14 +51,6 @@ export const openLogDirectory = () => openDirectory(logDirectoryPath()); * "userData" directory. This is the **primary** place applications are meant to * store user's data, e.g. various configuration files and saved state. * - * During development, our app name is "Electron", so this'd be, for example, - * `~/Library/Application Support/Electron` if we run using `yarn dev`. For the - * packaged production app, our app name is "ente", so this would be: - * - * - Windows: `%APPDATA%\ente`, e.g. `C:\Users\\AppData\Local\ente` - * - Linux: `~/.config/ente` - * - macOS: `~/Library/Application Support/ente` - * * Note that Chromium also stores the browser state, e.g. localStorage or disk * caches, in userData. * @@ -71,7 +63,6 @@ export const openLogDirectory = () => openDirectory(logDirectoryPath()); * "ente.log", it can be found at: * * - macOS: ~/Library/Logs/ente/ente.log (production) - * - macOS: ~/Library/Logs/Electron/ente.log (dev) * - Linux: ~/.config/ente/logs/ente.log * - Windows: %USERPROFILE%\AppData\Roaming\ente\logs\ente.log */ diff --git a/desktop/src/main/services/store.ts b/desktop/src/main/services/store.ts index 471928d76c..253c2cbf0c 100644 --- a/desktop/src/main/services/store.ts +++ b/desktop/src/main/services/store.ts @@ -18,10 +18,7 @@ export const clearStores = () => { * [Note: Safe storage keys] * * On macOS, `safeStorage` stores our data under a Keychain entry named - * " Safe Storage". Which resolves to: - * - * - Electron Safe Storage (dev) - * - ente Safe Storage (prod) + * " Safe Storage". In our case, "ente Safe Storage". */ export const saveEncryptionKey = (encryptionKey: string) => { const encryptedKey = safeStorage.encryptString(encryptionKey); diff --git a/desktop/src/preload.ts b/desktop/src/preload.ts index 537215c6b0..85475031d3 100644 --- a/desktop/src/preload.ts +++ b/desktop/src/preload.ts @@ -65,7 +65,7 @@ const selectDirectory = () => ipcRenderer.invoke("selectDirectory"); const logout = () => { watchRemoveListeners(); - ipcRenderer.send("logout"); + return ipcRenderer.invoke("logout"); }; const encryptionKey = () => ipcRenderer.invoke("encryptionKey"); diff --git a/desktop/thirdparty/next-electron-server b/desktop/thirdparty/next-electron-server deleted file mode 160000 index a88030295c..0000000000 --- a/desktop/thirdparty/next-electron-server +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a88030295c89dd8f43d9e3a45025678d95c78a45 diff --git a/mobile/lib/services/machine_learning/file_ml/remote_fileml_service.dart b/mobile/lib/services/machine_learning/file_ml/remote_fileml_service.dart index ec4e632b3e..eafbc6323d 100644 --- a/mobile/lib/services/machine_learning/file_ml/remote_fileml_service.dart +++ b/mobile/lib/services/machine_learning/file_ml/remote_fileml_service.dart @@ -1,7 +1,6 @@ import "dart:async"; import "dart:convert"; -import "package:computer/computer.dart"; import "package:logging/logging.dart"; import "package:photos/core/network/network.dart"; import "package:photos/db/files_db.dart"; @@ -22,15 +21,8 @@ class RemoteFileMLService { final _logger = Logger("RemoteFileMLService"); final _dio = NetworkClient.instance.enteDio; - final _computer = Computer.shared(); - late SharedPreferences _preferences; - - Completer? _syncStatus; - - void init(SharedPreferences prefs) { - _preferences = prefs; - } + void init(SharedPreferences prefs) {} Future putFileEmbedding(EnteFile file, FileMl fileML) async { final encryptionKey = getFileKey(file); diff --git a/mobile/lib/ui/viewer/file_details/faces_item_widget.dart b/mobile/lib/ui/viewer/file_details/faces_item_widget.dart index 0e31960fda..ed2fb0f12e 100644 --- a/mobile/lib/ui/viewer/file_details/faces_item_widget.dart +++ b/mobile/lib/ui/viewer/file_details/faces_item_widget.dart @@ -142,7 +142,7 @@ class _FacesItemWidgetState extends State { final faceWidgets = []; // await generation of the face crops here, so that the file info shows one central loading spinner - final test = await getRelevantFaceCrops(faces); + final _ = await getRelevantFaceCrops(faces); final faceCrops = getRelevantFaceCrops(faces); for (final Face face in faces) { diff --git a/mobile/lib/utils/local_settings.dart b/mobile/lib/utils/local_settings.dart index 8050c1d732..6b81e76971 100644 --- a/mobile/lib/utils/local_settings.dart +++ b/mobile/lib/utils/local_settings.dart @@ -86,7 +86,7 @@ class LocalSettings { //#region todo:(NG) remove this section, only needed for internal testing to see // if the OS stops the app during indexing - bool get remoteFetchEnabled => _prefs.getBool("remoteFetchEnabled") ?? false; + bool get remoteFetchEnabled => _prefs.getBool("remoteFetchEnabled") ?? true; Future toggleRemoteFetch() async { await _prefs.setBool("remoteFetchEnabled", !remoteFetchEnabled); } diff --git a/mobile/pubspec.yaml b/mobile/pubspec.yaml index ff29cf622b..6464496f58 100644 --- a/mobile/pubspec.yaml +++ b/mobile/pubspec.yaml @@ -12,7 +12,7 @@ description: ente photos application # Read more about iOS versioning at # https://developer.apple.com/library/archive/documentation/General/Reference/InfoPlistKeyReference/Articles/CoreFoundationKeys.html -version: 0.8.96+616 +version: 0.8.97+617 publish_to: none environment: diff --git a/mobile/thirdparty/flutter b/mobile/thirdparty/flutter deleted file mode 160000 index 367f9ea16b..0000000000 --- a/mobile/thirdparty/flutter +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 367f9ea16bfae1ca451b9cc27c1366870b187ae2 diff --git a/mobile/thirdparty/isar b/mobile/thirdparty/isar deleted file mode 160000 index 6643d064ab..0000000000 --- a/mobile/thirdparty/isar +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 6643d064abf22606b6c6a741ea873e4781115ef4 diff --git a/web/apps/photos/package.json b/web/apps/photos/package.json index a200c8ef74..8d515c07cd 100644 --- a/web/apps/photos/package.json +++ b/web/apps/photos/package.json @@ -27,7 +27,7 @@ "leaflet-defaulticon-compatibility": "^0.1.1", "localforage": "^1.9.0", "memoize-one": "^6.0.0", - "ml-matrix": "^6.10.4", + "ml-matrix": "^6.11", "otpauth": "^9.0.2", "p-debounce": "^4.0.0", "p-queue": "^7.1.0", @@ -42,7 +42,7 @@ "react-window": "^1.8.6", "sanitize-filename": "^1.6.3", "similarity-transformation": "^0.0.1", - "transformation-matrix": "^2.15.0", + "transformation-matrix": "^2.16", "uuid": "^9.0.1", "vscode-uri": "^3.0.7", "xml-js": "^1.6.11", diff --git a/web/apps/photos/src/pages/gallery/index.tsx b/web/apps/photos/src/pages/gallery/index.tsx index 9ade12fc5e..cb0ae1bf15 100644 --- a/web/apps/photos/src/pages/gallery/index.tsx +++ b/web/apps/photos/src/pages/gallery/index.tsx @@ -717,10 +717,10 @@ export default function Gallery() { await syncTrash(collections, setTrashedFiles); await syncEntities(); await syncMapEnabled(); - await syncCLIPEmbeddings(); const electron = globalThis.electron; - if (isInternalUserForML() && electron) { - await syncFaceEmbeddings(); + if (electron) { + await syncCLIPEmbeddings(); + if (isInternalUserForML()) await syncFaceEmbeddings(); } if (clipService.isPlatformSupported()) { void clipService.scheduleImageEmbeddingExtraction(); diff --git a/web/apps/photos/src/services/face/f-index.ts b/web/apps/photos/src/services/face/f-index.ts index 853cd15af5..39456f7ddd 100644 --- a/web/apps/photos/src/services/face/f-index.ts +++ b/web/apps/photos/src/services/face/f-index.ts @@ -2,7 +2,6 @@ import { FILE_TYPE } from "@/media/file-type"; import { blobCache } from "@/next/blob-cache"; import log from "@/next/log"; import { workerBridge } from "@/next/worker/worker-bridge"; -import { euclidean } from "hdbscan"; import { Matrix } from "ml-matrix"; import { Box, @@ -19,6 +18,13 @@ import type { } from "services/face/types"; import { defaultMLVersion } from "services/machineLearning/machineLearningService"; import { getSimilarityTransformation } from "similarity-transformation"; +import { + Matrix as TransformationMatrix, + applyToPoint, + compose, + scale, + translate, +} from "transformation-matrix"; import type { EnteFile } from "types/file"; import { fetchImageBitmap, getLocalFileImageBitmap } from "./file"; import { @@ -27,7 +33,6 @@ import { pixelRGBBilinear, warpAffineFloat32List, } from "./image"; -import { transformFaceDetections } from "./transform-box"; /** * Index faces in the given file. @@ -138,7 +143,7 @@ const indexFaces_ = async (enteFile: EnteFile, imageBitmap: ImageBitmap) => { /** * Detect faces in the given {@link imageBitmap}. * - * The model used is YOLO, running in an ONNX runtime. + * The model used is YOLOv5Face, running in an ONNX runtime. */ const detectFaces = async ( imageBitmap: ImageBitmap, @@ -149,16 +154,14 @@ const detectFaces = async ( const { yoloInput, yoloSize } = convertToYOLOInputFloat32ChannelsFirst(imageBitmap); const yoloOutput = await workerBridge.detectFaces(yoloInput); - const faces = faceDetectionsFromYOLOOutput(yoloOutput); + const faces = filterExtractDetectionsFromYOLOOutput(yoloOutput); const faceDetections = transformFaceDetections( faces, rect(yoloSize), rect(imageBitmap), ); - const maxFaceDistancePercent = Math.sqrt(2) / 100; - const maxFaceDistance = imageBitmap.width * maxFaceDistancePercent; - return removeDuplicateDetections(faceDetections, maxFaceDistance); + return naiveNonMaxSuppression(faceDetections, 0.4); }; /** @@ -214,14 +217,24 @@ const convertToYOLOInputFloat32ChannelsFirst = (imageBitmap: ImageBitmap) => { }; /** - * Extract detected faces from the YOLO's output. + * Extract detected faces from the YOLOv5Face's output. * * Only detections that exceed a minimum score are returned. * - * @param rows A Float32Array of shape [25200, 16], where each row - * represents a bounding box. + * @param rows A Float32Array of shape [25200, 16], where each row represents a + * face detection. + * + * YOLO detects a fixed number of faces, 25200, always from the input it is + * given. Each detection is a "row" of 16 bytes, containing the bounding box, + * score, and landmarks of the detection. + * + * We prune out detections with a score lower than our threshold. However, we + * will still be left with some overlapping detections of the same face: these + * we will deduplicate in {@link removeDuplicateDetections}. */ -const faceDetectionsFromYOLOOutput = (rows: Float32Array): FaceDetection[] => { +const filterExtractDetectionsFromYOLOOutput = ( + rows: Float32Array, +): FaceDetection[] => { const faces: FaceDetection[] = []; // Iterate over each row. for (let i = 0; i < rows.length; i += 16) { @@ -266,61 +279,121 @@ const faceDetectionsFromYOLOOutput = (rows: Float32Array): FaceDetection[] => { }; /** - * Removes duplicate face detections from an array of detections. + * Transform the given {@link faceDetections} from their coordinate system in + * which they were detected ({@link inBox}) back to the coordinate system of the + * original image ({@link toBox}). + */ +const transformFaceDetections = ( + faceDetections: FaceDetection[], + inBox: Box, + toBox: Box, +): FaceDetection[] => { + const transform = boxTransformationMatrix(inBox, toBox); + return faceDetections.map((f) => ({ + box: transformBox(f.box, transform), + landmarks: f.landmarks.map((p) => transformPoint(p, transform)), + probability: f.probability, + })); +}; + +const boxTransformationMatrix = ( + inBox: Box, + toBox: Box, +): TransformationMatrix => + compose( + translate(toBox.x, toBox.y), + scale(toBox.width / inBox.width, toBox.height / inBox.height), + ); + +const transformPoint = (point: Point, transform: TransformationMatrix) => { + const txdPoint = applyToPoint(transform, point); + return new Point(txdPoint.x, txdPoint.y); +}; + +const transformBox = (box: Box, transform: TransformationMatrix) => { + const topLeft = transformPoint(new Point(box.x, box.y), transform); + const bottomRight = transformPoint( + new Point(box.x + box.width, box.y + box.height), + transform, + ); + + return new Box({ + x: topLeft.x, + y: topLeft.y, + width: bottomRight.x - topLeft.x, + height: bottomRight.y - topLeft.y, + }); +}; + +/** + * Remove overlapping faces from an array of face detections through non-maximum + * suppression algorithm. * * This function sorts the detections by their probability in descending order, * then iterates over them. * - * For each detection, it calculates the Euclidean distance to all other - * detections. + * For each detection, it calculates the Intersection over Union (IoU) with all + * other detections. * - * If the distance is less than or equal to the specified threshold - * (`withinDistance`), the other detection is considered a duplicate and is + * If the IoU is greater than or equal to the specified threshold + * (`iouThreshold`), the other detection is considered overlapping and is * removed. * - * @param detections - An array of face detections to remove duplicates from. + * @param detections - An array of face detections to remove overlapping faces + * from. * - * @param withinDistance - The maximum Euclidean distance between two detections - * for them to be considered duplicates. + * @param iouThreshold - The minimum IoU between two detections for them to be + * considered overlapping. * - * @returns An array of face detections with duplicates removed. + * @returns An array of face detections with overlapping faces removed */ -const removeDuplicateDetections = ( +const naiveNonMaxSuppression = ( detections: FaceDetection[], - withinDistance: number, -) => { + iouThreshold: number, +): FaceDetection[] => { + // Sort the detections by score, the highest first. detections.sort((a, b) => b.probability - a.probability); - const dupIndices = new Set(); - for (let i = 0; i < detections.length; i++) { - if (dupIndices.has(i)) continue; - + // Loop through the detections and calculate the IOU. + for (let i = 0; i < detections.length - 1; i++) { for (let j = i + 1; j < detections.length; j++) { - if (dupIndices.has(j)) continue; - - const centeri = faceDetectionCenter(detections[i]); - const centerj = faceDetectionCenter(detections[j]); - const dist = euclidean( - [centeri.x, centeri.y], - [centerj.x, centerj.y], - ); - - if (dist <= withinDistance) dupIndices.add(j); + const iou = intersectionOverUnion(detections[i], detections[j]); + if (iou >= iouThreshold) { + detections.splice(j, 1); + j--; + } } } - return detections.filter((_, i) => !dupIndices.has(i)); + return detections; }; -const faceDetectionCenter = (detection: FaceDetection) => { - const center = new Point(0, 0); - // TODO-ML(LAURENS): first 4 landmarks is applicable to blazeface only this - // needs to consider eyes, nose and mouth landmarks to take center - detection.landmarks?.slice(0, 4).forEach((p) => { - center.x += p.x; - center.y += p.y; - }); - return new Point(center.x / 4, center.y / 4); +const intersectionOverUnion = (a: FaceDetection, b: FaceDetection): number => { + const intersectionMinX = Math.max(a.box.x, b.box.x); + const intersectionMinY = Math.max(a.box.y, b.box.y); + const intersectionMaxX = Math.min( + a.box.x + a.box.width, + b.box.x + b.box.width, + ); + const intersectionMaxY = Math.min( + a.box.y + a.box.height, + b.box.y + b.box.height, + ); + + const intersectionWidth = intersectionMaxX - intersectionMinX; + const intersectionHeight = intersectionMaxY - intersectionMinY; + + if (intersectionWidth < 0 || intersectionHeight < 0) { + return 0.0; // If boxes do not overlap, IoU is 0 + } + + const areaA = a.box.width * a.box.height; + const areaB = b.box.width * b.box.height; + + const intersectionArea = intersectionWidth * intersectionHeight; + const unionArea = areaA + areaB - intersectionArea; + + return intersectionArea / unionArea; }; const makeFaceID = ( @@ -398,12 +471,15 @@ const faceAlignmentUsingSimilarityTransform = ( const meanTranslation = simTransform.toMean.sub(0.5).mul(size); const centerMat = simTransform.fromMean.sub(meanTranslation); const center = new Point(centerMat.get(0, 0), centerMat.get(1, 0)); - const rotation = -Math.atan2( - simTransform.rotation.get(0, 1), - simTransform.rotation.get(0, 0), - ); - return { affineMatrix, center, size, rotation }; + const boundingBox = new Box({ + x: center.x - size / 2, + y: center.y - size / 2, + width: size, + height: size, + }); + + return { affineMatrix, boundingBox }; }; const convertToMobileFaceNetInput = ( @@ -678,35 +754,22 @@ const extractFaceCrop = ( imageBitmap: ImageBitmap, alignment: FaceAlignment, ): ImageBitmap => { - const alignmentBox = new Box({ - x: alignment.center.x - alignment.size / 2, - y: alignment.center.y - alignment.size / 2, - width: alignment.size, - height: alignment.size, - }); + // TODO-ML: This algorithm is different from what is used by the mobile app. + // Also, it needs to be something that can work fully using the embedding we + // receive from remote - the `alignment.boundingBox` will not be available + // to us in such cases. + const paddedBox = roundBox(enlargeBox(alignment.boundingBox, 1.5)); + const outputSize = { width: paddedBox.width, height: paddedBox.height }; - const padding = 0.25; - const scaleForPadding = 1 + padding * 2; - const paddedBox = roundBox(enlargeBox(alignmentBox, scaleForPadding)); + const maxDimension = 256; + const scale = Math.min( + maxDimension / paddedBox.width, + maxDimension / paddedBox.height, + ); - // TODO-ML(LAURENS): The rotation doesn't seem to be used? it's set to 0. - return cropWithRotation(imageBitmap, paddedBox, 0, 256); -}; - -const cropWithRotation = ( - imageBitmap: ImageBitmap, - cropBox: Box, - rotation: number, - maxDimension: number, -) => { - const box = roundBox(cropBox); - - const outputSize = { width: box.width, height: box.height }; - - const scale = Math.min(maxDimension / box.width, maxDimension / box.height); if (scale < 1) { - outputSize.width = Math.round(scale * box.width); - outputSize.height = Math.round(scale * box.height); + outputSize.width = Math.round(scale * paddedBox.width); + outputSize.height = Math.round(scale * paddedBox.height); } const offscreen = new OffscreenCanvas(outputSize.width, outputSize.height); @@ -714,7 +777,6 @@ const cropWithRotation = ( offscreenCtx.imageSmoothingQuality = "high"; offscreenCtx.translate(outputSize.width / 2, outputSize.height / 2); - rotation && offscreenCtx.rotate(rotation); const outputBox = new Box({ x: -outputSize.width / 2, @@ -723,7 +785,7 @@ const cropWithRotation = ( height: outputSize.height, }); - const enlargedBox = enlargeBox(box, 1.5); + const enlargedBox = enlargeBox(paddedBox, 1.5); const enlargedOutputBox = enlargeBox(outputBox, 1.5); offscreenCtx.drawImage( diff --git a/web/apps/photos/src/services/face/remote.ts b/web/apps/photos/src/services/face/remote.ts index fcd8775a9e..c0a5189bc1 100644 --- a/web/apps/photos/src/services/face/remote.ts +++ b/web/apps/photos/src/services/face/remote.ts @@ -20,16 +20,18 @@ export const putFaceEmbedding = async ( const comlinkCryptoWorker = await ComlinkCryptoWorker.getInstance(); const { file: encryptedEmbeddingData } = await comlinkCryptoWorker.encryptMetadata(serverMl, enteFile.key); - log.info( - `putEmbedding embedding to server for file: ${enteFile.metadata.title} fileID: ${enteFile.id}`, - ); - const res = await putEmbedding({ + // TODO-ML(MR): Do we need any of these fields + // log.info( + // `putEmbedding embedding to server for file: ${enteFile.metadata.title} fileID: ${enteFile.id}`, + // ); + /*const res =*/ await putEmbedding({ fileID: enteFile.id, encryptedEmbedding: encryptedEmbeddingData.encryptedData, decryptionHeader: encryptedEmbeddingData.decryptionHeader, model: "file-ml-clip-face", }); - log.info("putEmbedding response: ", res); + // TODO-ML(MR): Do we need any of these fields + // log.info("putEmbedding response: ", res); }; export interface FileML extends ServerFileMl { diff --git a/web/apps/photos/src/services/face/transform-box.ts b/web/apps/photos/src/services/face/transform-box.ts deleted file mode 100644 index 01fa2a9771..0000000000 --- a/web/apps/photos/src/services/face/transform-box.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { Box, Point } from "services/face/geom"; -import type { FaceDetection } from "services/face/types"; -// TODO-ML(LAURENS): Do we need two separate Matrix libraries? -// -// Keeping this in a separate file so that we can audit this. If these can be -// expressed using ml-matrix, then we can move this code to f-index.ts -import { - Matrix, - applyToPoint, - compose, - scale, - translate, -} from "transformation-matrix"; - -/** - * Transform the given {@link faceDetections} from their coordinate system in - * which they were detected ({@link inBox}) back to the coordinate system of the - * original image ({@link toBox}). - */ -export const transformFaceDetections = ( - faceDetections: FaceDetection[], - inBox: Box, - toBox: Box, -): FaceDetection[] => { - const transform = boxTransformationMatrix(inBox, toBox); - return faceDetections.map((f) => ({ - box: transformBox(f.box, transform), - landmarks: f.landmarks.map((p) => transformPoint(p, transform)), - probability: f.probability, - })); -}; - -const boxTransformationMatrix = (inBox: Box, toBox: Box): Matrix => - compose( - translate(toBox.x, toBox.y), - scale(toBox.width / inBox.width, toBox.height / inBox.height), - ); - -const transformPoint = (point: Point, transform: Matrix) => { - const txdPoint = applyToPoint(transform, point); - return new Point(txdPoint.x, txdPoint.y); -}; - -const transformBox = (box: Box, transform: Matrix) => { - const topLeft = transformPoint(new Point(box.x, box.y), transform); - const bottomRight = transformPoint( - new Point(box.x + box.width, box.y + box.height), - transform, - ); - - return new Box({ - x: topLeft.x, - y: topLeft.y, - width: bottomRight.x - topLeft.x, - height: bottomRight.y - topLeft.y, - }); -}; diff --git a/web/apps/photos/src/services/face/types.ts b/web/apps/photos/src/services/face/types.ts index e1fa32785f..423f6afb7b 100644 --- a/web/apps/photos/src/services/face/types.ts +++ b/web/apps/photos/src/services/face/types.ts @@ -8,13 +8,20 @@ export interface FaceDetection { } export interface FaceAlignment { - // TODO-ML(MR): remove affine matrix as rotation, size and center - // are simple to store and use, affine matrix adds complexity while getting crop + /** + * An affine transformation matrix (rotation, translation, scaling) to align + * the face extracted from the image. + */ affineMatrix: number[][]; - rotation: number; - // size and center is relative to image dimentions stored at mlFileData - size: number; - center: Point; + /** + * The bounding box of the transformed box. + * + * The affine transformation shifts the original detection box a new, + * transformed, box (possibily rotated). This property is the bounding box + * of that transformed box. It is in the coordinate system of the original, + * full, image on which the detection occurred. + */ + boundingBox: Box; } export interface Face { diff --git a/web/docs/dependencies.md b/web/docs/dependencies.md index f5082b9f34..2ff8e40172 100644 --- a/web/docs/dependencies.md +++ b/web/docs/dependencies.md @@ -177,12 +177,19 @@ some cases. ## Face search -- [matrix](https://github.com/mljs/matrix) and - [similarity-transformation](https://github.com/shaileshpandit/similarity-transformation-js) - are used during face alignment. - - [transformation-matrix](https://github.com/chrvadala/transformation-matrix) - is used during face detection. + is used for performing 2D affine transformations using transformation + matrices. It is used during face detection. + +- [matrix](https://github.com/mljs/matrix) is mathematical matrix abstraction. + It is used alongwith + [similarity-transformation](https://github.com/shaileshpandit/similarity-transformation-js) + during face alignment. + + > Note that while both `transformation-matrix` and `matrix` are "matrix" + > libraries, they have different foci and purposes: `transformation-matrix` + > provides affine transforms, while `matrix` is for performing computations + > on matrices, say inverting them or performing their decomposition. - [hdbscan](https://github.com/shaileshpandit/hdbscan-js) is used for face clustering. diff --git a/web/packages/next/locales/pt-BR/translation.json b/web/packages/next/locales/pt-BR/translation.json index a191a49272..5adb222bca 100644 --- a/web/packages/next/locales/pt-BR/translation.json +++ b/web/packages/next/locales/pt-BR/translation.json @@ -428,7 +428,7 @@ "USED": "usado", "YOU": "Você", "FAMILY": "Família", - "FREE": "grátis", + "FREE": "livre", "OF": "de", "WATCHED_FOLDERS": "Pastas monitoradas", "NO_FOLDERS_ADDED": "Nenhuma pasta adicionada ainda!", diff --git a/web/yarn.lock b/web/yarn.lock index a18a0a0dc8..bb12308316 100644 --- a/web/yarn.lock +++ b/web/yarn.lock @@ -3528,7 +3528,7 @@ ml-array-rescale@^1.3.7: ml-array-max "^1.2.4" ml-array-min "^1.2.3" -ml-matrix@^6.10.4: +ml-matrix@^6.11: version "6.11.0" resolved "https://registry.yarnpkg.com/ml-matrix/-/ml-matrix-6.11.0.tgz#3cf2260ef04cbb8e0e0425e71d200f5cbcf82772" integrity sha512-7jr9NmFRkaUxbKslfRu3aZOjJd2LkSitCGv+QH9PF0eJoEG7jIpjXra1Vw8/kgao8+kHCSsJONG6vfWmXQ+/Eg== @@ -4628,7 +4628,7 @@ tr46@~0.0.3: resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a" integrity sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw== -transformation-matrix@^2.15.0: +transformation-matrix@^2.16: version "2.16.1" resolved "https://registry.yarnpkg.com/transformation-matrix/-/transformation-matrix-2.16.1.tgz#4a2de06331b94ae953193d1b9a5ba002ec5f658a" integrity sha512-tdtC3wxVEuzU7X/ydL131Q3JU5cPMEn37oqVLITjRDSDsnSHVFzW2JiCLfZLIQEgWzZHdSy3J6bZzvKEN24jGA==