From d19d7ffe7997b82190c3ad87e8727fdf8566aa24 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Sat, 6 Apr 2024 03:49:03 +0530 Subject: [PATCH] [mob][face] Storage width/height along with area and visibility --- mobile/lib/face/db.dart | 8 ++++-- mobile/lib/face/db_fields.dart | 8 ++++++ mobile/lib/face/db_model_mappers.dart | 8 ++++++ mobile/lib/face/model/detection.dart | 1 - mobile/lib/face/model/face.dart | 25 +++++++++++++++++-- .../face_ml/face_ml_service.dart | 8 +++++- .../ui/viewer/file_details/face_widget.dart | 4 +-- 7 files changed, 54 insertions(+), 8 deletions(-) diff --git a/mobile/lib/face/db.dart b/mobile/lib/face/db.dart index a482f500cf..bb901a044c 100644 --- a/mobile/lib/face/db.dart +++ b/mobile/lib/face/db.dart @@ -272,12 +272,16 @@ class FaceMLDataDB { final List> maps = await db.query( facesTable, columns: [ - fileIDColumn, faceIDColumn, - faceDetectionColumn, + fileIDColumn, faceEmbeddingBlob, faceScore, + faceDetectionColumn, faceBlur, + imageHeight, + imageWidth, + faceArea, + faceVisibilityScore, mlVersionColumn, ], where: '$fileIDColumn = ?', diff --git a/mobile/lib/face/db_fields.dart b/mobile/lib/face/db_fields.dart index c62f33338a..2dc98ac1f3 100644 --- a/mobile/lib/face/db_fields.dart +++ b/mobile/lib/face/db_fields.dart @@ -8,6 +8,10 @@ const faceDetectionColumn = 'detection'; const faceEmbeddingBlob = 'eBlob'; const faceScore = 'score'; const faceBlur = 'blur'; +const faceArea = 'area'; +const faceVisibilityScore = 'visibility'; +const imageWidth = 'width'; +const imageHeight = 'height'; const faceClusterId = 'cluster_id'; const mlVersionColumn = 'ml_version'; @@ -18,6 +22,10 @@ const createFacesTable = '''CREATE TABLE IF NOT EXISTS $facesTable ( $faceEmbeddingBlob BLOB NOT NULL, $faceScore REAL NOT NULL, $faceBlur REAL NOT NULL DEFAULT $kLapacianDefault, + $imageHeight INTEGER NOT NULL DEFAULT 0, + $imageWidth INTEGER NOT NULL DEFAULT 0, + $faceArea INTEGER NOT NULL DEFAULT 0, + $faceVisibilityScore INTEGER NOT NULL DEFAULT -1, $mlVersionColumn INTEGER NOT NULL DEFAULT -1, PRIMARY KEY($fileIDColumn, $faceIDColumn) ); diff --git a/mobile/lib/face/db_model_mappers.dart b/mobile/lib/face/db_model_mappers.dart index a85cd750a2..4e33a0bfdb 100644 --- a/mobile/lib/face/db_model_mappers.dart +++ b/mobile/lib/face/db_model_mappers.dart @@ -35,6 +35,10 @@ Map mapRemoteToFaceDB(Face face) { faceScore: face.score, faceBlur: face.blur, mlVersionColumn: faceMlVersion, + faceArea: face.area(), + faceVisibilityScore: face.visibility, + imageWidth: face.fileInfo?.imageWidth ?? 0, + imageHeight: face.fileInfo?.imageHeight ?? 0, }; } @@ -46,5 +50,9 @@ Face mapRowToFace(Map row) { row[faceScore] as double, Detection.fromJson(json.decode(row[faceDetectionColumn] as String)), row[faceBlur] as double, + fileInfo: FileInfo( + imageWidth: row[imageWidth] as int, + imageHeight: row[imageHeight] as int, + ), ); } diff --git a/mobile/lib/face/model/detection.dart b/mobile/lib/face/model/detection.dart index 1ffaa1eb3d..22c25b98f6 100644 --- a/mobile/lib/face/model/detection.dart +++ b/mobile/lib/face/model/detection.dart @@ -43,7 +43,6 @@ class Detection { ); } - // TODO: iterate on better area calculation, potentially using actual indexing image dimensions instead of file metadata int getFaceArea(int imageWidth, int imageHeight) { return (box.width * imageWidth * box.height * imageHeight).toInt(); } diff --git a/mobile/lib/face/model/face.dart b/mobile/lib/face/model/face.dart index 0df0987dff..631eeb141c 100644 --- a/mobile/lib/face/model/face.dart +++ b/mobile/lib/face/model/face.dart @@ -1,6 +1,16 @@ import "package:photos/face/model/detection.dart"; import 'package:photos/services/machine_learning/face_ml/face_filtering/face_filtering_constants.dart'; +// FileInfo contains the image width and height of the image the face was detected in. +class FileInfo { + int? imageWidth; + int? imageHeight; + FileInfo({ + this.imageWidth, + this.imageHeight, + }); +} + class Face { final int fileID; final String faceID; @@ -8,6 +18,7 @@ class Face { Detection detection; final double score; final double blur; + FileInfo? fileInfo; bool get isBlurry => blur < kLaplacianThreshold; @@ -15,14 +26,24 @@ class Face { bool get isHighQuality => (!isBlurry) && hasHighScore; + int get visibility => detection.getVisibilityScore(); + + int area({int? w, int? h}) { + return detection.getFaceArea( + fileInfo?.imageWidth ?? w ?? 0, + fileInfo?.imageHeight ?? h ?? 0, + ); + } + Face( this.faceID, this.fileID, this.embedding, this.score, this.detection, - this.blur, - ); + this.blur, { + this.fileInfo, + }); factory Face.empty(int fileID, {bool error = false}) { return Face( diff --git a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart index 2581ac622a..979914a46e 100644 --- a/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart +++ b/mobile/lib/services/machine_learning/face_ml/face_ml_service.dart @@ -538,7 +538,13 @@ class FaceMlService { ), ); } else { - faces.addAll(fileMl.faceEmbedding.faces); + for (final f in fileMl.faceEmbedding.faces) { + f.fileInfo = FileInfo( + imageHeight: fileMl.height, + imageWidth: fileMl.width, + ); + faces.add(f); + } } remoteFileIdToVersion[fileMl.fileID] = fileMl.faceEmbedding.version; } diff --git a/mobile/lib/ui/viewer/file_details/face_widget.dart b/mobile/lib/ui/viewer/file_details/face_widget.dart index ef75870b12..4bd4a7bb59 100644 --- a/mobile/lib/ui/viewer/file_details/face_widget.dart +++ b/mobile/lib/ui/viewer/file_details/face_widget.dart @@ -170,13 +170,13 @@ class _FaceWidgetState extends State { ), if (kDebugMode) Text( - 'V: ${widget.face.detection.getVisibilityScore()}', + 'V: ${widget.face.visibility}', style: Theme.of(context).textTheme.bodySmall, maxLines: 1, ), if (kDebugMode) Text( - 'A: ${widget.face.detection.getFaceArea(widget.file.width, widget.file.height)}', + 'A: ${widget.face.area()}', style: Theme.of(context).textTheme.bodySmall, maxLines: 1, ),