From a44e932c84f4994090507f576fc2310a38e04bb8 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 29 May 2024 13:50:04 +0530 Subject: [PATCH 01/77] Plan --- web/apps/photos/src/services/face/indexer.ts | 23 ++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 web/apps/photos/src/services/face/indexer.ts diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts new file mode 100644 index 0000000000..6feba91ae1 --- /dev/null +++ b/web/apps/photos/src/services/face/indexer.ts @@ -0,0 +1,23 @@ +/** + * Face indexer + * + * This is class that drives the face indexing process, across all files that + * need to still be indexed. This usually runs in a Web Worker so as to not get + * in the way of the main thread. + * + * It operates in two modes - live indexing and backfill. + * + * In live indexing, any files that are being uploaded from the current client + * are provided to the indexer, which indexes them. This is more efficient since + * we already have the file's content at hand and do not have to download and + * decrypt it. + * + * In backfill, the indexer figures out if any of the user's files (irrespective + * of where they were uploaded from) still need to be indexed, and if so, + * downloads, decrypts and indexes them. + * + * Live indexing has higher priority, backfill runs otherwise. + * + * If nothing needs to be indexed, the indexer goes to sleep. + */ +export class Indexer {} From 2fb7ee01713434d791f4f00c22936bfbcc554869 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 29 May 2024 13:56:11 +0530 Subject: [PATCH 02/77] Sketch --- web/apps/photos/src/services/face/indexer.ts | 22 +++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index 6feba91ae1..8ac27ba1f8 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -1,5 +1,7 @@ +import type { EnteFile } from "types/file"; + /** - * Face indexer + * Face indexing orchestrator. * * This is class that drives the face indexing process, across all files that * need to still be indexed. This usually runs in a Web Worker so as to not get @@ -18,6 +20,20 @@ * * Live indexing has higher priority, backfill runs otherwise. * - * If nothing needs to be indexed, the indexer goes to sleep. + * If nothing needs to be indexed, the indexer goes to sleep for a while. */ -export class Indexer {} +export class FaceIndexer { + private liveItems: [file: File, enteFile: EnteFile][]; + + /** + * Add {@link file} associated with {@link enteFile} to the live indexing + * queue. + */ + enqueueFile(file: File, enteFile: EnteFile) { + this.liveItems.push([file, enteFile]); + } + + tick() { + + } +} From f8aa74979955ef57925058f59a7d3606669d6c40 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 29 May 2024 14:07:37 +0530 Subject: [PATCH 03/77] timeout --- web/apps/photos/src/services/face/indexer.ts | 32 ++++++++++++++++++-- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index 8ac27ba1f8..a2e15cbedf 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -1,4 +1,6 @@ import type { EnteFile } from "types/file"; +import { indexFaces } from "./f-index"; +import type { MlFileData } from "./types-old"; /** * Face indexing orchestrator. @@ -23,17 +25,41 @@ import type { EnteFile } from "types/file"; * If nothing needs to be indexed, the indexer goes to sleep for a while. */ export class FaceIndexer { - private liveItems: [file: File, enteFile: EnteFile][]; + /** Live indexing queue. */ + private liveItems: { file: File; enteFile: EnteFile }[]; + /** True when we are sleeping. */ + private isPaused = false; + /** Timeout for when the next time we will wake up. */ + private wakeTimeout: ReturnType | undefined; /** * Add {@link file} associated with {@link enteFile} to the live indexing * queue. */ enqueueFile(file: File, enteFile: EnteFile) { - this.liveItems.push([file, enteFile]); + this.liveItems.push({ file, enteFile }); + this.isPaused = false; + this.wakeUpIfNeeded() } - tick() { + private wakeUpIfNeeded() { + // Already awake + if (!this.wakeTimeout) return; + // Cancel the alarm, wake up now. + clearTimeout(this.wakeTimeout); + this.wakeTimeout = undefined; + this.tick(); + } + + private async tick() { + const item = this.liveItems.pop(); + let faceIndex: MlFileData | undefined; + if (item) { + faceIndex = await indexFaces(item.enteFile, item.file); + } else { + // backfill + } + console.log("indexed face", faceIndex); } } From 7f150d8dc724afb3ad575acf4bc28bc3f13b1ee2 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 29 May 2024 14:25:20 +0530 Subject: [PATCH 04/77] comp --- web/apps/photos/src/services/face/indexer.ts | 41 ++++++++++++++------ 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index a2e15cbedf..5a5df13c79 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -1,6 +1,8 @@ +import log from "@/next/log"; +import { wait } from "@/utils/promise"; import type { EnteFile } from "types/file"; +import { markIndexingFailed } from "./db"; import { indexFaces } from "./f-index"; -import type { MlFileData } from "./types-old"; /** * Face indexing orchestrator. @@ -27,7 +29,7 @@ import type { MlFileData } from "./types-old"; export class FaceIndexer { /** Live indexing queue. */ private liveItems: { file: File; enteFile: EnteFile }[]; - /** True when we are sleeping. */ + /** True when we have been paused externally. */ private isPaused = false; /** Timeout for when the next time we will wake up. */ private wakeTimeout: ReturnType | undefined; @@ -38,28 +40,45 @@ export class FaceIndexer { */ enqueueFile(file: File, enteFile: EnteFile) { this.liveItems.push({ file, enteFile }); - this.isPaused = false; - this.wakeUpIfNeeded() + this.wakeUpIfNeeded(); } private wakeUpIfNeeded() { - // Already awake + // If we were asked to pause, don't do anything. + if (this.isPaused) return; + // Already awake. if (!this.wakeTimeout) return; // Cancel the alarm, wake up now. clearTimeout(this.wakeTimeout); this.wakeTimeout = undefined; + // Get to work. this.tick(); } private async tick() { const item = this.liveItems.pop(); - let faceIndex: MlFileData | undefined; - if (item) { - faceIndex = await indexFaces(item.enteFile, item.file); - } else { - // backfill + if (!item) { + // TODO-ML: backfill instead if needed here. + if (!this.isPaused) { + this.wakeTimeout = setTimeout(() => { + this.wakeTimeout = undefined; + this.wakeUpIfNeeded(); + }, 30 * 1000); + } + return; } - console.log("indexed face", faceIndex); + const fileID = item.enteFile.id; + try { + const faceIndex = await indexFaces(item.enteFile, item.file); + } catch (e) { + log.error(`Failed to index faces in file ${fileID}`, e); + markIndexingFailed(item.enteFile.id); + } + + // Let the runloop drain. + await wait(0); + // Run again. + this.tick(); } } From c968cc3c4170e073944415ecd11d6a233b323225 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 29 May 2024 15:26:20 +0530 Subject: [PATCH 05/77] remote --- web/apps/photos/src/services/face/indexer.ts | 100 ++++-------------- .../src/services/face/indexer.worker.ts | 81 ++++++++++++++ 2 files changed, 102 insertions(+), 79 deletions(-) create mode 100644 web/apps/photos/src/services/face/indexer.worker.ts diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index 5a5df13c79..f028148cec 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -1,84 +1,26 @@ -import log from "@/next/log"; -import { wait } from "@/utils/promise"; -import type { EnteFile } from "types/file"; -import { markIndexingFailed } from "./db"; -import { indexFaces } from "./f-index"; +import { ComlinkWorker } from "@/next/worker/comlink-worker"; +import { type Remote } from "comlink"; +import { FaceIndexerWorker } from "./indexer.worker"; /** - * Face indexing orchestrator. - * - * This is class that drives the face indexing process, across all files that - * need to still be indexed. This usually runs in a Web Worker so as to not get - * in the way of the main thread. - * - * It operates in two modes - live indexing and backfill. - * - * In live indexing, any files that are being uploaded from the current client - * are provided to the indexer, which indexes them. This is more efficient since - * we already have the file's content at hand and do not have to download and - * decrypt it. - * - * In backfill, the indexer figures out if any of the user's files (irrespective - * of where they were uploaded from) still need to be indexed, and if so, - * downloads, decrypts and indexes them. - * - * Live indexing has higher priority, backfill runs otherwise. - * - * If nothing needs to be indexed, the indexer goes to sleep for a while. + * A promise for the lazily created singleton {@link FaceIndexerWorker} remote + * exposed by this module. */ -export class FaceIndexer { - /** Live indexing queue. */ - private liveItems: { file: File; enteFile: EnteFile }[]; - /** True when we have been paused externally. */ - private isPaused = false; - /** Timeout for when the next time we will wake up. */ - private wakeTimeout: ReturnType | undefined; +let _faceIndexerWorker: Promise>; - /** - * Add {@link file} associated with {@link enteFile} to the live indexing - * queue. - */ - enqueueFile(file: File, enteFile: EnteFile) { - this.liveItems.push({ file, enteFile }); - this.wakeUpIfNeeded(); - } +const createFaceIndexerComlinkWorker = () => + new ComlinkWorker( + "face-indexer", + new Worker(new URL("indexer.worker.ts", import.meta.url)), + ); - private wakeUpIfNeeded() { - // If we were asked to pause, don't do anything. - if (this.isPaused) return; - // Already awake. - if (!this.wakeTimeout) return; - // Cancel the alarm, wake up now. - clearTimeout(this.wakeTimeout); - this.wakeTimeout = undefined; - // Get to work. - this.tick(); - } - - private async tick() { - const item = this.liveItems.pop(); - if (!item) { - // TODO-ML: backfill instead if needed here. - if (!this.isPaused) { - this.wakeTimeout = setTimeout(() => { - this.wakeTimeout = undefined; - this.wakeUpIfNeeded(); - }, 30 * 1000); - } - return; - } - - const fileID = item.enteFile.id; - try { - const faceIndex = await indexFaces(item.enteFile, item.file); - } catch (e) { - log.error(`Failed to index faces in file ${fileID}`, e); - markIndexingFailed(item.enteFile.id); - } - - // Let the runloop drain. - await wait(0); - // Run again. - this.tick(); - } -} +/** + * Main thread interface to the face indexer. + * + * This function provides a promise that resolves to a lazily created singleton + * remote with a {@link FaceIndexerWorker} at the other end. + * + * For more details, see the documentation for {@link FaceIndexerWorker}. + */ +export const faceIndexerWorker = (): Promise> => + (_faceIndexerWorker ??= createFaceIndexerComlinkWorker().remote); diff --git a/web/apps/photos/src/services/face/indexer.worker.ts b/web/apps/photos/src/services/face/indexer.worker.ts new file mode 100644 index 0000000000..6ec4f2be94 --- /dev/null +++ b/web/apps/photos/src/services/face/indexer.worker.ts @@ -0,0 +1,81 @@ +import log from "@/next/log"; +import { wait } from "@/utils/promise"; +import type { EnteFile } from "types/file"; +import { markIndexingFailed } from "./db"; +import { indexFaces } from "./f-index"; + +/** + * Face indexing orchestrator. + * + * This is class that drives the face indexing process, across all files that + * need to still be indexed. This usually runs in a Web Worker so as to not get + * in the way of the main thread. + * + * It operates in two modes - live indexing and backfill. + * + * In live indexing, any files that are being uploaded from the current client + * are provided to the indexer, which indexes them. This is more efficient since + * we already have the file's content at hand and do not have to download and + * decrypt it. + * + * In backfill, the indexer figures out if any of the user's files (irrespective + * of where they were uploaded from) still need to be indexed, and if so, + * downloads, decrypts and indexes them. + * + * Live indexing has higher priority, backfill runs otherwise. + * + * If nothing needs to be indexed, the indexer goes to sleep for a while. + */ +export class FaceIndexerWorker { + /** Live indexing queue. */ + private liveItems: { file: File; enteFile: EnteFile }[]; + /** Timeout for when the next time we will wake up. */ + private wakeTimeout: ReturnType | undefined; + + /** + * Add {@link file} associated with {@link enteFile} to the live indexing + * queue. + */ + enqueueFile(file: File, enteFile: EnteFile) { + this.liveItems.push({ file, enteFile }); + this.wakeUpIfNeeded(); + } + + private wakeUpIfNeeded() { + // Already awake. + if (!this.wakeTimeout) return; + // Cancel the alarm, wake up now. + clearTimeout(this.wakeTimeout); + this.wakeTimeout = undefined; + // Get to work. + this.tick(); + } + + private async tick() { + console.log("tick"); + + const item = this.liveItems.pop(); + if (!item) { + // TODO-ML: backfill instead if needed here. + this.wakeTimeout = setTimeout(() => { + this.wakeTimeout = undefined; + this.wakeUpIfNeeded(); + }, 30 * 1000); + return; + } + + const fileID = item.enteFile.id; + try { + const faceIndex = await indexFaces(item.enteFile, item.file); + log.info(`faces in file ${fileID}`, faceIndex); + } catch (e) { + log.error(`Failed to index faces in file ${fileID}`, e); + markIndexingFailed(item.enteFile.id); + } + + // Let the runloop drain. + await wait(0); + // Run again. + this.tick(); + } +} From 9adc8126bb7120b38201840c63f05c3cc87d8157 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 29 May 2024 15:28:35 +0530 Subject: [PATCH 06/77] Rename --- web/apps/photos/src/services/face/indexer.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index f028148cec..f998e5ac4d 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -6,7 +6,7 @@ import { FaceIndexerWorker } from "./indexer.worker"; * A promise for the lazily created singleton {@link FaceIndexerWorker} remote * exposed by this module. */ -let _faceIndexerWorker: Promise>; +let _faceIndexer: Promise>; const createFaceIndexerComlinkWorker = () => new ComlinkWorker( @@ -19,8 +19,6 @@ const createFaceIndexerComlinkWorker = () => * * This function provides a promise that resolves to a lazily created singleton * remote with a {@link FaceIndexerWorker} at the other end. - * - * For more details, see the documentation for {@link FaceIndexerWorker}. */ -export const faceIndexerWorker = (): Promise> => - (_faceIndexerWorker ??= createFaceIndexerComlinkWorker().remote); +export const faceIndexer = (): Promise> => + (_faceIndexer ??= createFaceIndexerComlinkWorker().remote); From daf72d8ac6d72aeaf8b832be8e339e7b71b69962 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 29 May 2024 15:39:13 +0530 Subject: [PATCH 07/77] Tweak --- web/apps/photos/src/services/face/indexer.worker.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/web/apps/photos/src/services/face/indexer.worker.ts b/web/apps/photos/src/services/face/indexer.worker.ts index 6ec4f2be94..3f7ee822d0 100644 --- a/web/apps/photos/src/services/face/indexer.worker.ts +++ b/web/apps/photos/src/services/face/indexer.worker.ts @@ -7,16 +7,16 @@ import { indexFaces } from "./f-index"; /** * Face indexing orchestrator. * - * This is class that drives the face indexing process, across all files that - * need to still be indexed. This usually runs in a Web Worker so as to not get - * in the way of the main thread. + * This is class that drives the face indexing process across all files that + * need to still be indexed. It runs in a Web Worker so as to not get in the way + * of the main thread. * * It operates in two modes - live indexing and backfill. * * In live indexing, any files that are being uploaded from the current client - * are provided to the indexer, which indexes them. This is more efficient since - * we already have the file's content at hand and do not have to download and - * decrypt it. + * are provided to the indexer, which puts them in a queue and indexes them one + * by one. This is more efficient since we already have the file's content at + * hand and do not have to download and decrypt it. * * In backfill, the indexer figures out if any of the user's files (irrespective * of where they were uploaded from) still need to be indexed, and if so, From 6097f9d4ba844cd3a0b960557b85e4ba12339199 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 29 May 2024 18:40:27 +0530 Subject: [PATCH 08/77] wip --- web/apps/photos/src/services/face/indexer.ts | 18 +++++++++++++++++- .../services/machineLearning/mlWorkManager.ts | 4 ++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index f998e5ac4d..ba06e6f4a4 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -1,5 +1,7 @@ import { ComlinkWorker } from "@/next/worker/comlink-worker"; import { type Remote } from "comlink"; +import mlWorkManager from "services/machineLearning/mlWorkManager"; +import type { EnteFile } from "types/file"; import { FaceIndexerWorker } from "./indexer.worker"; /** @@ -20,5 +22,19 @@ const createFaceIndexerComlinkWorker = () => * This function provides a promise that resolves to a lazily created singleton * remote with a {@link FaceIndexerWorker} at the other end. */ -export const faceIndexer = (): Promise> => +const faceIndexer = (): Promise> => (_faceIndexer ??= createFaceIndexerComlinkWorker().remote); + +/** + * Add a newly uploaded file to the face indexing queue. + * + * @param enteFile The {@link EnteFile} that was uploaded. + * @param file + */ +export const indexFacesInFile = (enteFile: EnteFile, file: File) => { + if (!mlWorkManager.isMlSearchEnabled) return; + + faceIndexer().then((indexer) => { + indexer.enqueueFile(file, enteFile); + }); +}; diff --git a/web/apps/photos/src/services/machineLearning/mlWorkManager.ts b/web/apps/photos/src/services/machineLearning/mlWorkManager.ts index 0ec5f29541..3f7e9571e8 100644 --- a/web/apps/photos/src/services/machineLearning/mlWorkManager.ts +++ b/web/apps/photos/src/services/machineLearning/mlWorkManager.ts @@ -117,6 +117,10 @@ class MLWorkManager { ); } + public isMlSearchEnabled() { + return this.mlSearchEnabled; + } + public async setMlSearchEnabled(enabled: boolean) { if (!this.mlSearchEnabled && enabled) { log.info("Enabling MLWorkManager"); From ce1ba6112fff3951941e72eda793b76f94f2b918 Mon Sep 17 00:00:00 2001 From: ialexanderbrito Date: Wed, 29 May 2024 11:03:03 -0300 Subject: [PATCH 09/77] fix: icons error and new icon --- auth/assets/custom-icons/icons/configcat.svg | 6 +++--- auth/assets/custom-icons/icons/habbo.svg | 6 +++--- auth/assets/custom-icons/icons/local_wp.svg | 9 +++++++++ auth/assets/custom-icons/icons/mercado_livre.svg | 11 ++++++----- auth/assets/custom-icons/icons/sendgrid.svg | 6 +++--- 5 files changed, 24 insertions(+), 14 deletions(-) create mode 100644 auth/assets/custom-icons/icons/local_wp.svg diff --git a/auth/assets/custom-icons/icons/configcat.svg b/auth/assets/custom-icons/icons/configcat.svg index cfecd22b02..12f7e4e81a 100644 --- a/auth/assets/custom-icons/icons/configcat.svg +++ b/auth/assets/custom-icons/icons/configcat.svg @@ -1,7 +1,7 @@ - - + + - + diff --git a/auth/assets/custom-icons/icons/habbo.svg b/auth/assets/custom-icons/icons/habbo.svg index 746bcdb229..2866bc3638 100644 --- a/auth/assets/custom-icons/icons/habbo.svg +++ b/auth/assets/custom-icons/icons/habbo.svg @@ -1,7 +1,7 @@ - - + + - + diff --git a/auth/assets/custom-icons/icons/local_wp.svg b/auth/assets/custom-icons/icons/local_wp.svg new file mode 100644 index 0000000000..3dbe63b2af --- /dev/null +++ b/auth/assets/custom-icons/icons/local_wp.svg @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/auth/assets/custom-icons/icons/mercado_livre.svg b/auth/assets/custom-icons/icons/mercado_livre.svg index 7f4db5fd53..8eeb1b94b5 100644 --- a/auth/assets/custom-icons/icons/mercado_livre.svg +++ b/auth/assets/custom-icons/icons/mercado_livre.svg @@ -1,9 +1,10 @@ - - + + + - - + + - + diff --git a/auth/assets/custom-icons/icons/sendgrid.svg b/auth/assets/custom-icons/icons/sendgrid.svg index 1562adab90..3b65642792 100644 --- a/auth/assets/custom-icons/icons/sendgrid.svg +++ b/auth/assets/custom-icons/icons/sendgrid.svg @@ -1,7 +1,7 @@ - - + + - + From 08a073fc1b5a88956cf42662a349f3f6c2707241 Mon Sep 17 00:00:00 2001 From: ialexanderbrito Date: Wed, 29 May 2024 11:03:27 -0300 Subject: [PATCH 10/77] feat: add new icons and altnames --- auth/assets/custom-icons/_data/custom-icons.json | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/auth/assets/custom-icons/_data/custom-icons.json b/auth/assets/custom-icons/_data/custom-icons.json index b911183e88..94cc5b55cb 100644 --- a/auth/assets/custom-icons/_data/custom-icons.json +++ b/auth/assets/custom-icons/_data/custom-icons.json @@ -190,6 +190,15 @@ { "title": "Letterboxd" }, + { + "title": "Local", + "slug": "local_wp", + "altNames": [ + "LocalWP", + "Local WP", + "Local Wordpress" + ] + }, { "title": "Mastodon", "altNames": [ @@ -203,7 +212,12 @@ }, { "title": "Mercado Livre", - "slug": "mercado_livre" + "slug": "mercado_livre", + "altNames": [ + "Mercado Libre", + "MercadoLibre", + "MercadoLivre" + ] }, { "title": "Murena", From 72851397b1603b808cd5ab205c9c855622dc4c19 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 29 May 2024 20:00:38 +0530 Subject: [PATCH 11/77] wip --- web/apps/photos/src/services/face/indexer.ts | 82 +++++++++++++++++++ .../src/services/face/indexer.worker.ts | 75 +++-------------- 2 files changed, 92 insertions(+), 65 deletions(-) diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index ba06e6f4a4..e629d4533b 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -4,6 +4,88 @@ import mlWorkManager from "services/machineLearning/mlWorkManager"; import type { EnteFile } from "types/file"; import { FaceIndexerWorker } from "./indexer.worker"; +import log from "@/next/log"; +import { wait } from "@/utils/promise"; +import type { EnteFile } from "types/file"; +import { markIndexingFailed } from "./db"; +import { indexFaces } from "./f-index"; + +/** + * Face indexing orchestrator. + * + * This is class that drives the face indexing process across all files that + * need to still be indexed. It runs in a Web Worker so as to not get in the way + * of the main thread. + * + * It operates in two modes - live indexing and backfill. + * + * In live indexing, any files that are being uploaded from the current client + * are provided to the indexer, which puts them in a queue and indexes them one + * by one. This is more efficient since we already have the file's content at + * hand and do not have to download and decrypt it. + * + * In backfill, the indexer figures out if any of the user's files (irrespective + * of where they were uploaded from) still need to be indexed, and if so, + * downloads, decrypts and indexes them. + * + * Live indexing has higher priority, backfill runs otherwise. + * + * If nothing needs to be indexed, the indexer goes to sleep for a while. + */ +export class FaceIndexerWorker { + /** Live indexing queue. */ + private liveItems: { file: File; enteFile: EnteFile }[]; + /** Timeout for when the next time we will wake up. */ + private wakeTimeout: ReturnType | undefined; + + /** + * Add {@link file} associated with {@link enteFile} to the live indexing + * queue. + */ + enqueueFile(file: File, enteFile: EnteFile) { + this.liveItems.push({ file, enteFile }); + this.wakeUpIfNeeded(); + } + + private wakeUpIfNeeded() { + // Already awake. + if (!this.wakeTimeout) return; + // Cancel the alarm, wake up now. + clearTimeout(this.wakeTimeout); + this.wakeTimeout = undefined; + // Get to work. + this.tick(); + } + + private async tick() { + console.log("tick"); + + const item = this.liveItems.pop(); + if (!item) { + // TODO-ML: backfill instead if needed here. + this.wakeTimeout = setTimeout(() => { + this.wakeTimeout = undefined; + this.wakeUpIfNeeded(); + }, 30 * 1000); + return; + } + + const fileID = item.enteFile.id; + try { + const faceIndex = await indexFaces(item.enteFile, item.file); + log.info(`faces in file ${fileID}`, faceIndex); + } catch (e) { + log.error(`Failed to index faces in file ${fileID}`, e); + markIndexingFailed(item.enteFile.id); + } + + // Let the runloop drain. + await wait(0); + // Run again. + this.tick(); + } +} + /** * A promise for the lazily created singleton {@link FaceIndexerWorker} remote * exposed by this module. diff --git a/web/apps/photos/src/services/face/indexer.worker.ts b/web/apps/photos/src/services/face/indexer.worker.ts index 3f7ee822d0..61c62b4029 100644 --- a/web/apps/photos/src/services/face/indexer.worker.ts +++ b/web/apps/photos/src/services/face/indexer.worker.ts @@ -1,81 +1,26 @@ import log from "@/next/log"; -import { wait } from "@/utils/promise"; import type { EnteFile } from "types/file"; import { markIndexingFailed } from "./db"; import { indexFaces } from "./f-index"; /** - * Face indexing orchestrator. + * Index faces in a file, save the persist the results locally, and put them on + * remote. * - * This is class that drives the face indexing process across all files that - * need to still be indexed. It runs in a Web Worker so as to not get in the way - * of the main thread. - * - * It operates in two modes - live indexing and backfill. - * - * In live indexing, any files that are being uploaded from the current client - * are provided to the indexer, which puts them in a queue and indexes them one - * by one. This is more efficient since we already have the file's content at - * hand and do not have to download and decrypt it. - * - * In backfill, the indexer figures out if any of the user's files (irrespective - * of where they were uploaded from) still need to be indexed, and if so, - * downloads, decrypts and indexes them. - * - * Live indexing has higher priority, backfill runs otherwise. - * - * If nothing needs to be indexed, the indexer goes to sleep for a while. + * This class is instantiated in a Web Worker so as to not get in the way of the + * main thread. It could've been a bunch of free standing functions too, it is + * just a class for convenience of compatibility with how the rest of our + * comlink workers are structured. */ export class FaceIndexerWorker { - /** Live indexing queue. */ - private liveItems: { file: File; enteFile: EnteFile }[]; - /** Timeout for when the next time we will wake up. */ - private wakeTimeout: ReturnType | undefined; - - /** - * Add {@link file} associated with {@link enteFile} to the live indexing - * queue. - */ - enqueueFile(file: File, enteFile: EnteFile) { - this.liveItems.push({ file, enteFile }); - this.wakeUpIfNeeded(); - } - - private wakeUpIfNeeded() { - // Already awake. - if (!this.wakeTimeout) return; - // Cancel the alarm, wake up now. - clearTimeout(this.wakeTimeout); - this.wakeTimeout = undefined; - // Get to work. - this.tick(); - } - - private async tick() { - console.log("tick"); - - const item = this.liveItems.pop(); - if (!item) { - // TODO-ML: backfill instead if needed here. - this.wakeTimeout = setTimeout(() => { - this.wakeTimeout = undefined; - this.wakeUpIfNeeded(); - }, 30 * 1000); - return; - } - - const fileID = item.enteFile.id; + async index(enteFile: EnteFile, file: File | undefined) { + const fileID = enteFile.id; try { - const faceIndex = await indexFaces(item.enteFile, item.file); + const faceIndex = await indexFaces(enteFile, file); log.info(`faces in file ${fileID}`, faceIndex); } catch (e) { log.error(`Failed to index faces in file ${fileID}`, e); - markIndexingFailed(item.enteFile.id); + markIndexingFailed(enteFile.id); } - - // Let the runloop drain. - await wait(0); - // Run again. - this.tick(); } } From 4ce02fba93bb73e1b091d342b575e25be227c59a Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 29 May 2024 20:14:53 +0530 Subject: [PATCH 12/77] ll --- web/apps/photos/src/services/face/f-index.ts | 3 +- .../src/services/face/indexer.worker.ts | 29 ++++++++++++++++--- web/apps/photos/src/utils/file/index.ts | 8 +++++ 3 files changed, 35 insertions(+), 5 deletions(-) diff --git a/web/apps/photos/src/services/face/f-index.ts b/web/apps/photos/src/services/face/f-index.ts index 5e93f60bd6..a1a614f138 100644 --- a/web/apps/photos/src/services/face/f-index.ts +++ b/web/apps/photos/src/services/face/f-index.ts @@ -12,6 +12,7 @@ import { translate, } from "transformation-matrix"; import type { EnteFile } from "types/file"; +import { logIdentifier } from "utils/file"; import { saveFaceCrop } from "./crop"; import { fetchImageBitmap, getLocalFileImageBitmap } from "./file"; import { @@ -58,7 +59,7 @@ export const indexFaces = async (enteFile: EnteFile, localFile?: File) => { log.debug(() => { const nf = mlFile.faces?.length ?? 0; const ms = Date.now() - startTime; - return `Indexed ${nf} faces in file ${enteFile.id} (${ms} ms)`; + return `Indexed ${nf} faces in file ${logIdentifier(enteFile)} (${ms} ms)`; }); return mlFile; }; diff --git a/web/apps/photos/src/services/face/indexer.worker.ts b/web/apps/photos/src/services/face/indexer.worker.ts index 61c62b4029..84250dd703 100644 --- a/web/apps/photos/src/services/face/indexer.worker.ts +++ b/web/apps/photos/src/services/face/indexer.worker.ts @@ -1,6 +1,7 @@ import log from "@/next/log"; import type { EnteFile } from "types/file"; -import { markIndexingFailed } from "./db"; +import { logIdentifier } from "utils/file"; +import { closeFaceDBConnectionsIfNeeded, markIndexingFailed } from "./db"; import { indexFaces } from "./f-index"; /** @@ -13,14 +14,34 @@ import { indexFaces } from "./f-index"; * comlink workers are structured. */ export class FaceIndexerWorker { + /* + * Index faces in a file, save the persist the results locally, and put them + * on remote. + * + * @param enteFile The {@link EnteFile} to index. + * + * @param file If the file is one which is being uploaded from the current + * client, then we will also have access to the file's content. In such + * cases, pass a web {@link File} object to use that its data directly for + * face indexing. If this is not provided, then the file's contents will be + * downloaded and decrypted from remote. + */ async index(enteFile: EnteFile, file: File | undefined) { - const fileID = enteFile.id; + const f = logIdentifier(enteFile); try { const faceIndex = await indexFaces(enteFile, file); - log.info(`faces in file ${fileID}`, faceIndex); + log.info(`faces in file ${f}`, faceIndex); } catch (e) { - log.error(`Failed to index faces in file ${fileID}`, e); + log.error(`Failed to index faces in file ${f}`, e); markIndexingFailed(enteFile.id); } } + + /** + * Calls {@link closeFaceDBConnectionsIfNeeded} to close any open + * connections to the face DB from the web worker's context. + */ + closeFaceDB() { + closeFaceDBConnectionsIfNeeded(); + } } diff --git a/web/apps/photos/src/utils/file/index.ts b/web/apps/photos/src/utils/file/index.ts index 3a349abea7..c15cca63c0 100644 --- a/web/apps/photos/src/utils/file/index.ts +++ b/web/apps/photos/src/utils/file/index.ts @@ -81,6 +81,14 @@ class ModuleState { const moduleState = new ModuleState(); +/** + * @returns a string to use as an identifier when logging information about the + * given {@link enteFile}. The returned string contains the file name (for ease + * of debugging) and the file ID (for exactness). + */ +export const logIdentifier = (enteFile: EnteFile) => + `${enteFile.metadata.title ?? "-"} (${enteFile.id})`; + export async function getUpdatedEXIFFileForDownload( fileReader: FileReader, file: EnteFile, From 3b8ab89647713b7a8f8340f4396b9b7f122290cc Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 29 May 2024 20:27:57 +0530 Subject: [PATCH 13/77] w --- web/apps/photos/src/services/face/indexer.ts | 111 ++++++++++--------- 1 file changed, 60 insertions(+), 51 deletions(-) diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index e629d4533b..661aa65288 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -1,49 +1,54 @@ -import { ComlinkWorker } from "@/next/worker/comlink-worker"; -import { type Remote } from "comlink"; -import mlWorkManager from "services/machineLearning/mlWorkManager"; -import type { EnteFile } from "types/file"; -import { FaceIndexerWorker } from "./indexer.worker"; - import log from "@/next/log"; +import { ComlinkWorker } from "@/next/worker/comlink-worker"; import { wait } from "@/utils/promise"; +import { type Remote } from "comlink"; import type { EnteFile } from "types/file"; import { markIndexingFailed } from "./db"; import { indexFaces } from "./f-index"; +import { FaceIndexerWorker } from "./indexer.worker"; +import mlWorkManager from "services/machineLearning/mlWorkManager"; /** * Face indexing orchestrator. * - * This is class that drives the face indexing process across all files that - * need to still be indexed. It runs in a Web Worker so as to not get in the way - * of the main thread. + * This module exposes a singleton instance of this class which drives the face + * indexing process on the user's library. * - * It operates in two modes - live indexing and backfill. + * The indexer operates in two modes - live indexing and backfill. * - * In live indexing, any files that are being uploaded from the current client + * When live indexing, any files that are being uploaded from the current client * are provided to the indexer, which puts them in a queue and indexes them one * by one. This is more efficient since we already have the file's content at * hand and do not have to download and decrypt it. * - * In backfill, the indexer figures out if any of the user's files (irrespective - * of where they were uploaded from) still need to be indexed, and if so, - * downloads, decrypts and indexes them. + * When backfilling, the indexer figures out if any of the user's files + * (irrespective of where they were uploaded from) still need to be indexed, and + * if so, downloads, decrypts and indexes them. * - * Live indexing has higher priority, backfill runs otherwise. - * - * If nothing needs to be indexed, the indexer goes to sleep for a while. + * Live indexing has higher priority, backfilling runs otherwise. If nothing + * remains to be indexed, the indexer goes to sleep for a while. */ -export class FaceIndexerWorker { +class FaceIndexer { /** Live indexing queue. */ - private liveItems: { file: File; enteFile: EnteFile }[]; + private liveItems: { enteFile: EnteFile; file: File | undefined }[]; /** Timeout for when the next time we will wake up. */ private wakeTimeout: ReturnType | undefined; /** - * Add {@link file} associated with {@link enteFile} to the live indexing - * queue. + * Add a file to the live indexing queue. + * + * @param enteFile An {@link EnteFile} that should be indexed. + * + * @param file The contents of {@link enteFile} as a web {@link File} + * object, if available. */ - enqueueFile(file: File, enteFile: EnteFile) { - this.liveItems.push({ file, enteFile }); + enqueueFile(enteFile: EnteFile, file: File | undefined) { + // If face indexing is not enabled, don't enqueue anything. Later on if + // the user turns on face indexing these files will get indexed as part + // of the backfilling anyway, the live indexing is just an optimization. + if (!mlWorkManager.isMlSearchEnabled) return; + + this.liveItems.push({ enteFile, file }); this.wakeUpIfNeeded(); } @@ -57,6 +62,20 @@ export class FaceIndexerWorker { this.tick(); } + /** + * A promise for the lazily created singleton {@link FaceIndexerWorker} remote + * exposed by this module. + */ + _faceIndexer: Promise>; + /** + * Main thread interface to the face indexer. + * + * This function provides a promise that resolves to a lazily created singleton + * remote with a {@link FaceIndexerWorker} at the other end. + */ + faceIndexer = (): Promise> => + (this._faceIndexer ??= createFaceIndexerComlinkWorker().remote); + private async tick() { console.log("tick"); @@ -84,39 +103,29 @@ export class FaceIndexerWorker { // Run again. this.tick(); } + + /** + * Add a newly uploaded file to the face indexing queue. + * + * @param enteFile The {@link EnteFile} that was uploaded. + * @param file + */ + /* + indexFacesInFile = (enteFile: EnteFile, file: File) => { + if (!mlWorkManager.isMlSearchEnabled) return; + + faceIndexer().then((indexer) => { + indexer.enqueueFile(file, enteFile); + }); + }; + */ } -/** - * A promise for the lazily created singleton {@link FaceIndexerWorker} remote - * exposed by this module. - */ -let _faceIndexer: Promise>; +/** The singleton instance of {@link FaceIndexer}. */ +export default new FaceIndexer(); const createFaceIndexerComlinkWorker = () => new ComlinkWorker( "face-indexer", new Worker(new URL("indexer.worker.ts", import.meta.url)), ); - -/** - * Main thread interface to the face indexer. - * - * This function provides a promise that resolves to a lazily created singleton - * remote with a {@link FaceIndexerWorker} at the other end. - */ -const faceIndexer = (): Promise> => - (_faceIndexer ??= createFaceIndexerComlinkWorker().remote); - -/** - * Add a newly uploaded file to the face indexing queue. - * - * @param enteFile The {@link EnteFile} that was uploaded. - * @param file - */ -export const indexFacesInFile = (enteFile: EnteFile, file: File) => { - if (!mlWorkManager.isMlSearchEnabled) return; - - faceIndexer().then((indexer) => { - indexer.enqueueFile(file, enteFile); - }); -}; From 61fb9cf544be5c25f15052d5ead4bd2217332d35 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 09:22:53 +0530 Subject: [PATCH 14/77] Flow via the new path --- web/apps/photos/src/services/face/indexer.ts | 2 +- web/apps/photos/src/services/face/indexer.worker.ts | 1 + .../src/services/machineLearning/machineLearningService.ts | 6 ++++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index 661aa65288..75330d86ed 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -2,11 +2,11 @@ import log from "@/next/log"; import { ComlinkWorker } from "@/next/worker/comlink-worker"; import { wait } from "@/utils/promise"; import { type Remote } from "comlink"; +import mlWorkManager from "services/machineLearning/mlWorkManager"; import type { EnteFile } from "types/file"; import { markIndexingFailed } from "./db"; import { indexFaces } from "./f-index"; import { FaceIndexerWorker } from "./indexer.worker"; -import mlWorkManager from "services/machineLearning/mlWorkManager"; /** * Face indexing orchestrator. diff --git a/web/apps/photos/src/services/face/indexer.worker.ts b/web/apps/photos/src/services/face/indexer.worker.ts index 84250dd703..d7b025fa45 100644 --- a/web/apps/photos/src/services/face/indexer.worker.ts +++ b/web/apps/photos/src/services/face/indexer.worker.ts @@ -31,6 +31,7 @@ export class FaceIndexerWorker { try { const faceIndex = await indexFaces(enteFile, file); log.info(`faces in file ${f}`, faceIndex); + return faceIndex; } catch (e) { log.error(`Failed to index faces in file ${f}`, e); markIndexingFailed(enteFile.id); diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index f871584743..c00089d59b 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -5,11 +5,11 @@ import mlIDbStorage, { ML_SEARCH_CONFIG_NAME, type MinimalPersistedFileData, } from "services/face/db-old"; +import { FaceIndexerWorker } from "services/face/indexer.worker"; import { putFaceEmbedding } from "services/face/remote"; import { getLocalFiles } from "services/fileService"; import { EnteFile } from "types/file"; import { isInternalUserForML } from "utils/user"; -import { indexFaces } from "../face/f-index"; export const defaultMLVersion = 1; @@ -346,7 +346,9 @@ class MachineLearningService { return oldMlFile; } - const newMlFile = await indexFaces(enteFile, localFile); + const worker = new FaceIndexerWorker(); + + const newMlFile = await worker.index(enteFile, localFile); await putFaceEmbedding(enteFile, newMlFile, userAgent); await mlIDbStorage.putFile(newMlFile); return newMlFile; From 8a1acc756efc9ed635a068d52e7dbc04b27cca5a Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 09:32:45 +0530 Subject: [PATCH 15/77] Move the put to worker --- .../photos/src/services/face/indexer.worker.ts | 18 ++++++++++++++---- .../machineLearning/machineLearningService.ts | 4 +--- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/web/apps/photos/src/services/face/indexer.worker.ts b/web/apps/photos/src/services/face/indexer.worker.ts index d7b025fa45..b4b9bc08c7 100644 --- a/web/apps/photos/src/services/face/indexer.worker.ts +++ b/web/apps/photos/src/services/face/indexer.worker.ts @@ -1,8 +1,10 @@ import log from "@/next/log"; +import { putFaceEmbedding } from "services/face/remote"; import type { EnteFile } from "types/file"; import { logIdentifier } from "utils/file"; import { closeFaceDBConnectionsIfNeeded, markIndexingFailed } from "./db"; import { indexFaces } from "./f-index"; +import type { MlFileData } from "./types-old"; /** * Index faces in a file, save the persist the results locally, and put them on @@ -26,16 +28,24 @@ export class FaceIndexerWorker { * face indexing. If this is not provided, then the file's contents will be * downloaded and decrypted from remote. */ - async index(enteFile: EnteFile, file: File | undefined) { + async index(enteFile: EnteFile, file: File | undefined, userAgent: string) { const f = logIdentifier(enteFile); + + let faceIndex: MlFileData; try { - const faceIndex = await indexFaces(enteFile, file); - log.info(`faces in file ${f}`, faceIndex); - return faceIndex; + faceIndex = await indexFaces(enteFile, file); + log.debug(() => ({ f, faceIndex })); } catch (e) { + // Mark indexing as having failed only if the indexing itself + // failed, not if there were subsequent failures (like when trying + // to put the result to remote or save it to the local face DB). log.error(`Failed to index faces in file ${f}`, e); markIndexingFailed(enteFile.id); + throw e; } + + await putFaceEmbedding(enteFile, faceIndex, userAgent); + return faceIndex; } /** diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index c00089d59b..ef12c0891b 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -6,7 +6,6 @@ import mlIDbStorage, { type MinimalPersistedFileData, } from "services/face/db-old"; import { FaceIndexerWorker } from "services/face/indexer.worker"; -import { putFaceEmbedding } from "services/face/remote"; import { getLocalFiles } from "services/fileService"; import { EnteFile } from "types/file"; import { isInternalUserForML } from "utils/user"; @@ -348,8 +347,7 @@ class MachineLearningService { const worker = new FaceIndexerWorker(); - const newMlFile = await worker.index(enteFile, localFile); - await putFaceEmbedding(enteFile, newMlFile, userAgent); + const newMlFile = await worker.index(enteFile, localFile, userAgent); await mlIDbStorage.putFile(newMlFile); return newMlFile; } From 54654159ffece0c0a48aebaa831e03fef3f4aaea Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 09:36:26 +0530 Subject: [PATCH 16/77] Remove unused --- web/apps/photos/src/services/face/types-old.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/web/apps/photos/src/services/face/types-old.ts b/web/apps/photos/src/services/face/types-old.ts index 66eec9cf55..c64fc88ee7 100644 --- a/web/apps/photos/src/services/face/types-old.ts +++ b/web/apps/photos/src/services/face/types-old.ts @@ -33,8 +33,6 @@ export interface Face { blurValue?: number; embedding?: Float32Array; - - personId?: number; } export interface MlFileData { From bae4c65ab3a2ae003c8ba9bc544a2b2d5406f66c Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 09:43:22 +0530 Subject: [PATCH 17/77] Pull out the alignment --- web/apps/photos/src/services/face/crop.ts | 11 ++++-- web/apps/photos/src/services/face/f-index.ts | 37 +++++++++++++------ .../photos/src/services/face/types-old.ts | 19 ---------- 3 files changed, 33 insertions(+), 34 deletions(-) diff --git a/web/apps/photos/src/services/face/crop.ts b/web/apps/photos/src/services/face/crop.ts index faf7f0ac9b..8a09c1b8e6 100644 --- a/web/apps/photos/src/services/face/crop.ts +++ b/web/apps/photos/src/services/face/crop.ts @@ -1,9 +1,14 @@ import { blobCache } from "@/next/blob-cache"; +import type { FaceAlignment } from "./f-index"; import type { Box } from "./types"; -import type { Face, FaceAlignment } from "./types-old"; +import type { Face } from "./types-old"; -export const saveFaceCrop = async (imageBitmap: ImageBitmap, face: Face) => { - const faceCrop = extractFaceCrop(imageBitmap, face.alignment); +export const saveFaceCrop = async ( + imageBitmap: ImageBitmap, + face: Face, + alignment: FaceAlignment, +) => { + const faceCrop = extractFaceCrop(imageBitmap, alignment); const blob = await imageBitmapToBlob(faceCrop); faceCrop.close(); diff --git a/web/apps/photos/src/services/face/f-index.ts b/web/apps/photos/src/services/face/f-index.ts index a1a614f138..714f89c09f 100644 --- a/web/apps/photos/src/services/face/f-index.ts +++ b/web/apps/photos/src/services/face/f-index.ts @@ -22,12 +22,7 @@ import { warpAffineFloat32List, } from "./image"; import type { Box, Dimensions } from "./types"; -import type { - Face, - FaceAlignment, - FaceDetection, - MlFileData, -} from "./types-old"; +import type { Face, FaceDetection, MlFileData } from "./types-old"; /** * Index faces in the given file. @@ -109,11 +104,12 @@ const indexFaces_ = async (enteFile: EnteFile, imageBitmap: ImageBitmap) => { const alignments: FaceAlignment[] = []; for (const face of mlFile.faces) { - const alignment = faceAlignment(face.detection); - face.alignment = alignment; + const alignment = computeFaceAlignment(face.detection); alignments.push(alignment); - await saveFaceCrop(imageBitmap, face); + // This step is not really part of the indexing pipeline, we just do + // it here since we have already computed the face alignment. + await saveFaceCrop(imageBitmap, face, alignment); } const alignedFacesData = convertToMobileFaceNetInput( @@ -393,13 +389,30 @@ const makeFaceID = ( return [`${fileID}`, xMin, yMin, xMax, yMax].join("_"); }; +export interface FaceAlignment { + /** + * An affine transformation matrix (rotation, translation, scaling) to align + * the face extracted from the image. + */ + affineMatrix: number[][]; + /** + * The bounding box of the transformed box. + * + * The affine transformation shifts the original detection box a new, + * transformed, box (possibily rotated). This property is the bounding box + * of that transformed box. It is in the coordinate system of the original, + * full, image on which the detection occurred. + */ + boundingBox: Box; +} + /** * Compute and return an {@link FaceAlignment} for the given face detection. * * @param faceDetection A geometry indicating a face detected in an image. */ -const faceAlignment = (faceDetection: FaceDetection): FaceAlignment => - faceAlignmentUsingSimilarityTransform( +const computeFaceAlignment = (faceDetection: FaceDetection): FaceAlignment => + computeFaceAlignmentUsingSimilarityTransform( faceDetection, normalizeLandmarks(idealMobileFaceNetLandmarks, mobileFaceNetFaceSize), ); @@ -422,7 +435,7 @@ const normalizeLandmarks = ( ): [number, number][] => landmarks.map(([x, y]) => [x / faceSize, y / faceSize]); -const faceAlignmentUsingSimilarityTransform = ( +const computeFaceAlignmentUsingSimilarityTransform = ( faceDetection: FaceDetection, alignedLandmarks: [number, number][], ): FaceAlignment => { diff --git a/web/apps/photos/src/services/face/types-old.ts b/web/apps/photos/src/services/face/types-old.ts index c64fc88ee7..ad6ef68743 100644 --- a/web/apps/photos/src/services/face/types-old.ts +++ b/web/apps/photos/src/services/face/types-old.ts @@ -7,29 +7,10 @@ export interface FaceDetection { probability?: number; } -export interface FaceAlignment { - /** - * An affine transformation matrix (rotation, translation, scaling) to align - * the face extracted from the image. - */ - affineMatrix: number[][]; - /** - * The bounding box of the transformed box. - * - * The affine transformation shifts the original detection box a new, - * transformed, box (possibily rotated). This property is the bounding box - * of that transformed box. It is in the coordinate system of the original, - * full, image on which the detection occurred. - */ - boundingBox: Box; -} - export interface Face { fileId: number; detection: FaceDetection; id: string; - - alignment?: FaceAlignment; blurValue?: number; embedding?: Float32Array; From ddddc09226b378bf00541c04e83564dc78415b06 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 10:02:57 +0530 Subject: [PATCH 18/77] New --- web/apps/photos/src/services/face/indexer.ts | 41 +++++++++++++++++++ web/apps/photos/src/services/searchService.ts | 5 ++- 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index 75330d86ed..d9ce6066a7 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -5,6 +5,7 @@ import { type Remote } from "comlink"; import mlWorkManager from "services/machineLearning/mlWorkManager"; import type { EnteFile } from "types/file"; import { markIndexingFailed } from "./db"; +import type { IndexStatus } from "./db-old"; import { indexFaces } from "./f-index"; import { FaceIndexerWorker } from "./indexer.worker"; @@ -129,3 +130,43 @@ const createFaceIndexerComlinkWorker = () => "face-indexer", new Worker(new URL("indexer.worker.ts", import.meta.url)), ); + +export interface FaceIndexingStatus { + /** + * Which phase we are in within the indexing pipeline when viewed across the + * user's entire library: + * + * - "scheduled": There are files we know of that have not been indexed. + * + * - "indexing": The face indexer is currently running. + * + * - "clustering": All files we know of have been indexed, and we are now + * clustering the faces that were found. + * + * - "done": Face indexing and clustering is complete for the user's + * library. + */ + phase: "scheduled" | "indexing" | "clustering" | "done"; + outOfSyncFilesExists: boolean; + nSyncedFiles: number; + nTotalFiles: number; + localFilesSynced: boolean; + peopleIndexSynced: boolean; +} + +export const convertToNewInterface = (indexStatus: IndexStatus) => { + let phase: string; + if (!indexStatus.localFilesSynced) { + phase = "scheduled"; + } else if (indexStatus.outOfSyncFilesExists) { + phase = "indexing"; + } else if (!indexStatus.peopleIndexSynced) { + phase = "clustering"; + } else { + phase = "done"; + } + return { + ...indexStatus, + phase, + }; +}; diff --git a/web/apps/photos/src/services/searchService.ts b/web/apps/photos/src/services/searchService.ts index b48778f690..472441ba66 100644 --- a/web/apps/photos/src/services/searchService.ts +++ b/web/apps/photos/src/services/searchService.ts @@ -22,6 +22,7 @@ import { getFormattedDate } from "utils/search"; import { clipService, computeClipMatchScore } from "./clip-service"; import { localCLIPEmbeddings } from "./embeddingService"; import { getLatestEntities } from "./entityService"; +import { convertToNewInterface } from "./face/indexer"; import locationSearchService, { City } from "./locationSearchService"; const DIGITS = new Set(["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"]); @@ -175,7 +176,9 @@ export async function getAllPeopleSuggestion(): Promise> { export async function getIndexStatusSuggestion(): Promise { try { - const indexStatus = await mlIDbStorage.getIndexStatus(defaultMLVersion); + const indexStatus0 = + await mlIDbStorage.getIndexStatus(defaultMLVersion); + const indexStatus = convertToNewInterface(indexStatus0); let label; if (!indexStatus.localFilesSynced) { From cbdd82f6c053f7d50cd1e5ccefac5104d93fd2ea Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 10:06:53 +0530 Subject: [PATCH 19/77] Use --- web/apps/photos/src/services/searchService.ts | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/web/apps/photos/src/services/searchService.ts b/web/apps/photos/src/services/searchService.ts index 472441ba66..e189899476 100644 --- a/web/apps/photos/src/services/searchService.ts +++ b/web/apps/photos/src/services/searchService.ts @@ -181,16 +181,21 @@ export async function getIndexStatusSuggestion(): Promise { const indexStatus = convertToNewInterface(indexStatus0); let label; - if (!indexStatus.localFilesSynced) { - label = t("INDEXING_SCHEDULED"); - } else if (indexStatus.outOfSyncFilesExists) { - label = t("ANALYZING_PHOTOS", { - indexStatus, - }); - } else if (!indexStatus.peopleIndexSynced) { - label = t("INDEXING_PEOPLE", { indexStatus }); - } else { - label = t("INDEXING_DONE", { indexStatus }); + switch (indexStatus.phase) { + case "scheduled": + label = t("INDEXING_SCHEDULED"); + break; + case "indexing": + label = t("ANALYZING_PHOTOS", { + indexStatus, + }); + break; + case "clustering": + label = t("INDEXING_PEOPLE", { indexStatus }); + break; + case "done": + label = t("INDEXING_DONE", { indexStatus }); + break; } return { From aa353b57e806c26b3cbbe35cce74cb125bd66420 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 10:07:48 +0530 Subject: [PATCH 20/77] Prune new API --- web/apps/photos/src/services/face/indexer.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index d9ce6066a7..7fa6c2b874 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -147,11 +147,8 @@ export interface FaceIndexingStatus { * library. */ phase: "scheduled" | "indexing" | "clustering" | "done"; - outOfSyncFilesExists: boolean; nSyncedFiles: number; nTotalFiles: number; - localFilesSynced: boolean; - peopleIndexSynced: boolean; } export const convertToNewInterface = (indexStatus: IndexStatus) => { From 85785f75432b8f5e9b5ad851a8eb650aa0f8b6aa Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 10:20:57 +0530 Subject: [PATCH 21/77] Doc --- web/apps/photos/src/services/face/indexer.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index 7fa6c2b874..1b1a01d6ed 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -147,7 +147,9 @@ export interface FaceIndexingStatus { * library. */ phase: "scheduled" | "indexing" | "clustering" | "done"; + /** The number of files that have already been indexed. */ nSyncedFiles: number; + /** The total number of files that are eligible for indexing. */ nTotalFiles: number; } From f6bd99386e1dd9d5ab51d683f7f1136f9e5f3442 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 10:28:40 +0530 Subject: [PATCH 22/77] t --- .../components/Search/SearchBar/searchInput/MenuWithPeople.tsx | 3 +-- web/apps/photos/src/services/searchService.ts | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/web/apps/photos/src/components/Search/SearchBar/searchInput/MenuWithPeople.tsx b/web/apps/photos/src/components/Search/SearchBar/searchInput/MenuWithPeople.tsx index aaca1c3906..fd073af643 100644 --- a/web/apps/photos/src/components/Search/SearchBar/searchInput/MenuWithPeople.tsx +++ b/web/apps/photos/src/components/Search/SearchBar/searchInput/MenuWithPeople.tsx @@ -5,7 +5,6 @@ import { t } from "i18next"; import { AppContext } from "pages/_app"; import { useContext } from "react"; import { components } from "react-select"; -import { IndexStatus } from "services/face/db-old"; import { Suggestion, SuggestionType } from "types/search"; const { Menu } = components; @@ -35,7 +34,7 @@ const MenuWithPeople = (props) => { (o) => o.type === SuggestionType.INDEX_STATUS, )[0] as Suggestion; - const indexStatus = indexStatusSuggestion?.value as IndexStatus; + const indexStatus = indexStatusSuggestion?.value; return ( diff --git a/web/apps/photos/src/services/searchService.ts b/web/apps/photos/src/services/searchService.ts index e189899476..159bd7525c 100644 --- a/web/apps/photos/src/services/searchService.ts +++ b/web/apps/photos/src/services/searchService.ts @@ -180,7 +180,7 @@ export async function getIndexStatusSuggestion(): Promise { await mlIDbStorage.getIndexStatus(defaultMLVersion); const indexStatus = convertToNewInterface(indexStatus0); - let label; + let label: string; switch (indexStatus.phase) { case "scheduled": label = t("INDEXING_SCHEDULED"); From 3c0d82279c4e61589de70f9b74bc3791ade21efe Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 10:35:02 +0530 Subject: [PATCH 23/77] Wrap --- web/apps/photos/src/services/face/indexer.ts | 10 +++++++++- web/apps/photos/src/services/searchService.ts | 7 ++----- web/apps/photos/src/types/search/index.ts | 4 ++-- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index 1b1a01d6ed..c2188a3921 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -2,6 +2,8 @@ import log from "@/next/log"; import { ComlinkWorker } from "@/next/worker/comlink-worker"; import { wait } from "@/utils/promise"; import { type Remote } from "comlink"; +import mlIDbStorage from "services/face/db-old"; +import { defaultMLVersion } from "services/machineLearning/machineLearningService"; import mlWorkManager from "services/machineLearning/mlWorkManager"; import type { EnteFile } from "types/file"; import { markIndexingFailed } from "./db"; @@ -153,8 +155,14 @@ export interface FaceIndexingStatus { nTotalFiles: number; } +export const faceIndexingStatus = async (): Promise => { + const indexStatus0 = await mlIDbStorage.getIndexStatus(defaultMLVersion); + const indexStatus = convertToNewInterface(indexStatus0); + return indexStatus; +}; + export const convertToNewInterface = (indexStatus: IndexStatus) => { - let phase: string; + let phase: FaceIndexingStatus["phase"]; if (!indexStatus.localFilesSynced) { phase = "scheduled"; } else if (indexStatus.outOfSyncFilesExists) { diff --git a/web/apps/photos/src/services/searchService.ts b/web/apps/photos/src/services/searchService.ts index 159bd7525c..ded48069f3 100644 --- a/web/apps/photos/src/services/searchService.ts +++ b/web/apps/photos/src/services/searchService.ts @@ -4,7 +4,6 @@ import * as chrono from "chrono-node"; import { t } from "i18next"; import mlIDbStorage from "services/face/db-old"; import type { Person } from "services/face/people"; -import { defaultMLVersion } from "services/machineLearning/machineLearningService"; import { Collection } from "types/collection"; import { EntityType, LocationTag, LocationTagData } from "types/entity"; import { EnteFile } from "types/file"; @@ -22,7 +21,7 @@ import { getFormattedDate } from "utils/search"; import { clipService, computeClipMatchScore } from "./clip-service"; import { localCLIPEmbeddings } from "./embeddingService"; import { getLatestEntities } from "./entityService"; -import { convertToNewInterface } from "./face/indexer"; +import { faceIndexingStatus } from "./face/indexer"; import locationSearchService, { City } from "./locationSearchService"; const DIGITS = new Set(["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"]); @@ -176,9 +175,7 @@ export async function getAllPeopleSuggestion(): Promise> { export async function getIndexStatusSuggestion(): Promise { try { - const indexStatus0 = - await mlIDbStorage.getIndexStatus(defaultMLVersion); - const indexStatus = convertToNewInterface(indexStatus0); + const indexStatus = await faceIndexingStatus(); let label: string; switch (indexStatus.phase) { diff --git a/web/apps/photos/src/types/search/index.ts b/web/apps/photos/src/types/search/index.ts index 3f3de9b460..adeb03d3aa 100644 --- a/web/apps/photos/src/types/search/index.ts +++ b/web/apps/photos/src/types/search/index.ts @@ -1,5 +1,5 @@ import { FILE_TYPE } from "@/media/file-type"; -import { IndexStatus } from "services/face/db-old"; +import type { FaceIndexingStatus } from "services/face/indexer"; import type { Person } from "services/face/people"; import { City } from "services/locationSearchService"; import { LocationTagData } from "types/entity"; @@ -31,7 +31,7 @@ export interface Suggestion { | DateValue | number[] | Person - | IndexStatus + | FaceIndexingStatus | LocationTagData | City | FILE_TYPE From d9200f470363f8f1fd63d271d44bb4f7c0a863de Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 10:55:46 +0530 Subject: [PATCH 24/77] Outline --- web/apps/photos/src/services/face/indexer.ts | 27 ++++++++++++++++++- .../machineLearning/machineLearningService.ts | 4 +++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index c2188a3921..d14b8d484c 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -3,7 +3,9 @@ import { ComlinkWorker } from "@/next/worker/comlink-worker"; import { wait } from "@/utils/promise"; import { type Remote } from "comlink"; import mlIDbStorage from "services/face/db-old"; -import { defaultMLVersion } from "services/machineLearning/machineLearningService"; +import machineLearningService, { + defaultMLVersion, +} from "services/machineLearning/machineLearningService"; import mlWorkManager from "services/machineLearning/mlWorkManager"; import type { EnteFile } from "types/file"; import { markIndexingFailed } from "./db"; @@ -156,8 +158,31 @@ export interface FaceIndexingStatus { } export const faceIndexingStatus = async (): Promise => { + const isSyncing = machineLearningService.isSyncing; + const [indexableCount, indexedCount] = [0, 0]; + + let phase: FaceIndexingStatus["phase"]; + if (indexableCount > 0 && indexableCount > indexedCount) { + if (!isSyncing) { + phase = "scheduled"; + } else { + phase = "indexing"; + } + } else { + phase = "done"; + } + + const indexingStatus = { + phase, + nTotalFiles: indexableCount, + nSyncedFiles: indexedCount, + }; + const indexStatus0 = await mlIDbStorage.getIndexStatus(defaultMLVersion); const indexStatus = convertToNewInterface(indexStatus0); + + log.debug(() => ({ indexStatus, indexingStatus })); + return indexStatus; }; diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index ef12c0891b..cbde414341 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -78,6 +78,8 @@ class MachineLearningService { private localSyncContext: Promise; private syncContext: Promise; + public isSyncing = false; + public async sync( token: string, userID: number, @@ -192,6 +194,7 @@ class MachineLearningService { } private async syncFiles(syncContext: MLSyncContext) { + this.isSyncing = true; try { const functions = syncContext.outOfSyncFiles.map( (outOfSyncfile) => async () => { @@ -211,6 +214,7 @@ class MachineLearningService { syncContext.error = error; } await syncContext.syncQueue.onIdle(); + this.isSyncing = false; // TODO: In case syncJob has to use multiple ml workers // do in same transaction with each file update From 896de6279473f91057b0e649498776c6f02bd5a1 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 11:05:04 +0530 Subject: [PATCH 25/77] Get counts from db --- web/apps/photos/src/services/face/db.ts | 14 ++++++++++++++ web/apps/photos/src/services/face/indexer.ts | 4 ++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/web/apps/photos/src/services/face/db.ts b/web/apps/photos/src/services/face/db.ts index 1128395237..85bb322056 100644 --- a/web/apps/photos/src/services/face/db.ts +++ b/web/apps/photos/src/services/face/db.ts @@ -197,6 +197,20 @@ export const addFileEntry = async (fileID: number) => { return tx.done; }; +/** + * Return the count of files that can be, and that have been, indexed. + */ +export const indexableAndIndexedCounts = async () => { + const db = await faceDB(); + const tx = db.transaction(["file-status", "face-index"], "readonly"); + const indexableCount = await tx + .objectStore("file-status") + .index("isIndexable") + .count(IDBKeyRange.only(1)); + const indexedCount = await tx.objectStore("face-index").count(); + return { indexableCount, indexedCount }; +}; + /** * Return a list of fileIDs that need to be indexed. * diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index d14b8d484c..a0a400b1e2 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -8,7 +8,7 @@ import machineLearningService, { } from "services/machineLearning/machineLearningService"; import mlWorkManager from "services/machineLearning/mlWorkManager"; import type { EnteFile } from "types/file"; -import { markIndexingFailed } from "./db"; +import { indexableAndIndexedCounts, markIndexingFailed } from "./db"; import type { IndexStatus } from "./db-old"; import { indexFaces } from "./f-index"; import { FaceIndexerWorker } from "./indexer.worker"; @@ -159,7 +159,7 @@ export interface FaceIndexingStatus { export const faceIndexingStatus = async (): Promise => { const isSyncing = machineLearningService.isSyncing; - const [indexableCount, indexedCount] = [0, 0]; + const { indexableCount, indexedCount } = await indexableAndIndexedCounts(); let phase: FaceIndexingStatus["phase"]; if (indexableCount > 0 && indexableCount > indexedCount) { From ab61fee8de181d32de82b9159589889a207565d4 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 11:06:31 +0530 Subject: [PATCH 26/77] simpl --- web/apps/photos/src/services/face/indexer.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index a0a400b1e2..90252720fc 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -162,7 +162,7 @@ export const faceIndexingStatus = async (): Promise => { const { indexableCount, indexedCount } = await indexableAndIndexedCounts(); let phase: FaceIndexingStatus["phase"]; - if (indexableCount > 0 && indexableCount > indexedCount) { + if (indexedCount < indexableCount) { if (!isSyncing) { phase = "scheduled"; } else { From 35090a6cdd303ab487e08e96ab4c82f4afa2ae0b Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 11:21:03 +0530 Subject: [PATCH 27/77] No clustering yet --- .../components/PhotoViewer/FileInfo/index.tsx | 6 +- .../photos/src/components/ml/PeopleList.tsx | 67 +------------------ 2 files changed, 5 insertions(+), 68 deletions(-) diff --git a/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx b/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx index a6d37ccf49..324928e9e6 100644 --- a/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx +++ b/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx @@ -11,7 +11,7 @@ import { Box, DialogProps, Link, Stack, styled } from "@mui/material"; import { Chip } from "components/Chip"; import { EnteDrawer } from "components/EnteDrawer"; import Titlebar from "components/Titlebar"; -import { PhotoPeopleList, UnidentifiedFaces } from "components/ml/PeopleList"; +import { UnidentifiedFaces } from "components/ml/PeopleList"; import LinkButton from "components/pages/gallery/LinkButton"; import { t } from "i18next"; import { AppContext } from "pages/_app"; @@ -332,10 +332,10 @@ export function FileInfo({ {appContext.mlSearchEnabled && ( <> - + /> */} >([]); - - useEffect(() => { - let didCancel = false; - - async function updateFaceImages() { - log.info("calling getPeopleList"); - const startTime = Date.now(); - const people = await getPeopleList(props.file); - log.info(`getPeopleList ${Date.now() - startTime} ms`); - log.info(`getPeopleList done, didCancel: ${didCancel}`); - !didCancel && setPeople(people); - } - - updateFaceImages(); - - return () => { - didCancel = true; - }; - }, [props.file, props.updateMLDataIndex]); - - if (people.length === 0) return <>; - - return ( -
- {t("PEOPLE")} - -
- ); +export function PhotoPeopleList() { + return <>; } export function UnidentifiedFaces(props: { @@ -180,40 +151,6 @@ const FaceCropImageView: React.FC = ({ faceID }) => { ); }; -async function getPeopleList(file: EnteFile): Promise { - let startTime = Date.now(); - const mlFileData = await mlIDbStorage.getFile(file.id); - log.info( - "getPeopleList:mlFilesStore:getItem", - Date.now() - startTime, - "ms", - ); - if (!mlFileData?.faces || mlFileData.faces.length < 1) { - return []; - } - - const peopleIds = mlFileData.faces - .filter((f) => f.personId !== null && f.personId !== undefined) - .map((f) => f.personId); - if (!peopleIds || peopleIds.length < 1) { - return []; - } - // log.info("peopleIds: ", peopleIds); - startTime = Date.now(); - const peoplePromises = peopleIds.map( - (p) => mlIDbStorage.getPerson(p) as Promise, - ); - const peopleList = await Promise.all(peoplePromises); - log.info( - "getPeopleList:mlPeopleStore:getItems", - Date.now() - startTime, - "ms", - ); - // log.info("peopleList: ", peopleList); - - return peopleList; -} - async function getUnidentifiedFaces(file: EnteFile): Promise<{ id: string }[]> { const mlFileData = await mlIDbStorage.getFile(file.id); From c3f6ecbf6afabb42824ed79dcb35689a050eb30d Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 11:30:23 +0530 Subject: [PATCH 28/77] Prune --- .../components/PhotoViewer/FileInfo/index.tsx | 12 +----- .../photos/src/components/ml/PeopleList.tsx | 40 ++++++++----------- 2 files changed, 18 insertions(+), 34 deletions(-) diff --git a/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx b/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx index 324928e9e6..5a17f43d17 100644 --- a/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx +++ b/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx @@ -96,8 +96,6 @@ export function FileInfo({ const [parsedExifData, setParsedExifData] = useState>(); const [showExif, setShowExif] = useState(false); - // eslint-disable-next-line @typescript-eslint/no-unused-vars - const [updateMLDataIndex, setUpdateMLDataIndex] = useState(0); const openExif = () => setShowExif(true); const closeExif = () => setShowExif(false); @@ -332,14 +330,8 @@ export function FileInfo({ {appContext.mlSearchEnabled && ( <> - {/* */} - + {/* */} + )} diff --git a/web/apps/photos/src/components/ml/PeopleList.tsx b/web/apps/photos/src/components/ml/PeopleList.tsx index 26c5c80556..def73dffd5 100644 --- a/web/apps/photos/src/components/ml/PeopleList.tsx +++ b/web/apps/photos/src/components/ml/PeopleList.tsx @@ -66,35 +66,29 @@ export const PeopleList = React.memo((props: PeopleListProps) => { export interface PhotoPeopleListProps extends PeopleListPropsBase { file: EnteFile; - updateMLDataIndex: number; } export function PhotoPeopleList() { return <>; } -export function UnidentifiedFaces(props: { - file: EnteFile; - updateMLDataIndex: number; -}) { +export function UnidentifiedFaces({ file }: { file: EnteFile }) { const [faces, setFaces] = useState<{ id: string }[]>([]); useEffect(() => { let didCancel = false; - async function updateFaceImages() { - const faces = await getUnidentifiedFaces(props.file); + (async () => { + const faces = await unidentifiedFaceIDs(file); !didCancel && setFaces(faces); - } - - updateFaceImages(); + })(); return () => { didCancel = true; }; - }, [props.file, props.updateMLDataIndex]); + }, [file]); - if (!faces || faces.length === 0) return <>; + if (faces.length == 0) return <>; return ( <> @@ -102,12 +96,11 @@ export function UnidentifiedFaces(props: { {t("UNIDENTIFIED_FACES")} - {faces && - faces.map((face, index) => ( - - - - ))} + {faces.map((face) => ( + + + + ))} ); @@ -151,10 +144,9 @@ const FaceCropImageView: React.FC = ({ faceID }) => { ); }; -async function getUnidentifiedFaces(file: EnteFile): Promise<{ id: string }[]> { +const unidentifiedFaceIDs = async ( + file: EnteFile, +): Promise<{ id: string }[]> => { const mlFileData = await mlIDbStorage.getFile(file.id); - - return mlFileData?.faces?.filter( - (f) => f.personId === null || f.personId === undefined, - ); -} + return mlFileData?.faces; +}; From 3c92349054f990eb137b86ecf9582d7ac91540c2 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 11:33:09 +0530 Subject: [PATCH 29/77] Move --- web/apps/photos/src/components/ml/PeopleList.tsx | 9 +-------- web/apps/photos/src/services/face/indexer.ts | 13 ++++++++++++- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/web/apps/photos/src/components/ml/PeopleList.tsx b/web/apps/photos/src/components/ml/PeopleList.tsx index def73dffd5..d44d100612 100644 --- a/web/apps/photos/src/components/ml/PeopleList.tsx +++ b/web/apps/photos/src/components/ml/PeopleList.tsx @@ -3,7 +3,7 @@ import { Skeleton, styled } from "@mui/material"; import { Legend } from "components/PhotoViewer/styledComponents/Legend"; import { t } from "i18next"; import React, { useEffect, useState } from "react"; -import mlIDbStorage from "services/face/db-old"; +import { unidentifiedFaceIDs } from "services/face/indexer"; import type { Person } from "services/face/people"; import { EnteFile } from "types/file"; @@ -143,10 +143,3 @@ const FaceCropImageView: React.FC = ({ faceID }) => { ); }; - -const unidentifiedFaceIDs = async ( - file: EnteFile, -): Promise<{ id: string }[]> => { - const mlFileData = await mlIDbStorage.getFile(file.id); - return mlFileData?.faces; -}; diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index 90252720fc..5d5d0f56c5 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -186,7 +186,7 @@ export const faceIndexingStatus = async (): Promise => { return indexStatus; }; -export const convertToNewInterface = (indexStatus: IndexStatus) => { +const convertToNewInterface = (indexStatus: IndexStatus) => { let phase: FaceIndexingStatus["phase"]; if (!indexStatus.localFilesSynced) { phase = "scheduled"; @@ -202,3 +202,14 @@ export const convertToNewInterface = (indexStatus: IndexStatus) => { phase, }; }; + +/** + * Return the IDs of all the faces in the given {@link enteFile} that are not + * associated with a person cluster. + */ +export const unidentifiedFaceIDs = async ( + enteFile: EnteFile, +): Promise<{ id: string }[]> => { + const mlFileData = await mlIDbStorage.getFile(enteFile.id); + return mlFileData?.faces ?? []; +}; From 321422e9151d39ad32bc9ad547f71bb47bcaa380 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 11:34:50 +0530 Subject: [PATCH 30/77] No clustering yet --- web/apps/photos/src/services/searchService.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/web/apps/photos/src/services/searchService.ts b/web/apps/photos/src/services/searchService.ts index ded48069f3..be7f574a7b 100644 --- a/web/apps/photos/src/services/searchService.ts +++ b/web/apps/photos/src/services/searchService.ts @@ -2,7 +2,6 @@ import { FILE_TYPE } from "@/media/file-type"; import log from "@/next/log"; import * as chrono from "chrono-node"; import { t } from "i18next"; -import mlIDbStorage from "services/face/db-old"; import type { Person } from "services/face/people"; import { Collection } from "types/collection"; import { EntityType, LocationTag, LocationTagData } from "types/entity"; @@ -435,7 +434,7 @@ function convertSuggestionToSearchQuery(option: Suggestion): Search { } async function getAllPeople(limit: number = undefined) { - let people: Array = await mlIDbStorage.getAllPeople(); + let people: Array = []; // await mlIDbStorage.getAllPeople(); // await mlPeopleStore.iterate((person) => { // people.push(person); // }); From 403cc3cca0d3309560770578770871b6e72d03a9 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 11:46:36 +0530 Subject: [PATCH 31/77] New --- web/apps/photos/src/services/face/indexer.ts | 29 ++++++++++++++++++- .../machineLearning/machineLearningService.ts | 12 ++------ 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index 5d5d0f56c5..7436986144 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -2,12 +2,14 @@ import log from "@/next/log"; import { ComlinkWorker } from "@/next/worker/comlink-worker"; import { wait } from "@/utils/promise"; import { type Remote } from "comlink"; -import mlIDbStorage from "services/face/db-old"; +import mlIDbStorage, { ML_SEARCH_CONFIG_NAME } from "services/face/db-old"; import machineLearningService, { + DEFAULT_ML_SEARCH_CONFIG, defaultMLVersion, } from "services/machineLearning/machineLearningService"; import mlWorkManager from "services/machineLearning/mlWorkManager"; import type { EnteFile } from "types/file"; +import { isInternalUserForML } from "utils/user"; import { indexableAndIndexedCounts, markIndexingFailed } from "./db"; import type { IndexStatus } from "./db-old"; import { indexFaces } from "./f-index"; @@ -213,3 +215,28 @@ export const unidentifiedFaceIDs = async ( const mlFileData = await mlIDbStorage.getFile(enteFile.id); return mlFileData?.faces ?? []; }; + +/** + * Return true if the user has enabled face indexing in the app's settings. + * + * This setting is persisted locally (in local storage) and is not synced with + * remote. There is a separate setting, "faceSearchEnabled" that is synced with + * remote, but that tracks whether or not the user has enabled face search once + * on any client. This {@link isFaceIndexingEnabled} property, on the other + * hand, denotes whether or not indexing is enabled on the current client. + */ +export const isFaceIndexingEnabled = () => { + if (isInternalUserForML()) { + return mlIDbStorage.getConfig( + ML_SEARCH_CONFIG_NAME, + DEFAULT_ML_SEARCH_CONFIG, + ); + } + // Force disabled for everyone else while we finalize it to avoid redundant + // reindexing for users. + return DEFAULT_ML_SEARCH_CONFIG; +}; + +export const setIsFaceIndexingEnabled = (enabled: boolean) => { + return mlIDbStorage.putConfig(ML_SEARCH_CONFIG_NAME, { enabled }); +}; diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index cbde414341..f952a3202e 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -5,10 +5,10 @@ import mlIDbStorage, { ML_SEARCH_CONFIG_NAME, type MinimalPersistedFileData, } from "services/face/db-old"; +import { isFaceIndexingEnabled } from "services/face/indexer"; import { FaceIndexerWorker } from "services/face/indexer.worker"; import { getLocalFiles } from "services/fileService"; import { EnteFile } from "types/file"; -import { isInternalUserForML } from "utils/user"; export const defaultMLVersion = 1; @@ -25,15 +25,7 @@ export const DEFAULT_ML_SEARCH_CONFIG: MLSearchConfig = { }; export async function getMLSearchConfig() { - if (isInternalUserForML()) { - return mlIDbStorage.getConfig( - ML_SEARCH_CONFIG_NAME, - DEFAULT_ML_SEARCH_CONFIG, - ); - } - // Force disabled for everyone else while we finalize it to avoid redundant - // reindexing for users. - return DEFAULT_ML_SEARCH_CONFIG; + return isFaceIndexingEnabled(); } export async function updateMLSearchConfig(newConfig: MLSearchConfig) { From 6be42225c2a80a70b2844be1ba3c2a9fcfaf9110 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 11:49:36 +0530 Subject: [PATCH 32/77] Bypass --- web/apps/photos/src/pages/_app.tsx | 15 ++++++--------- web/apps/photos/src/services/face/indexer.ts | 7 ++++--- .../machineLearning/machineLearningService.ts | 5 ----- 3 files changed, 10 insertions(+), 17 deletions(-) diff --git a/web/apps/photos/src/pages/_app.tsx b/web/apps/photos/src/pages/_app.tsx index a816d0b462..4157634aa5 100644 --- a/web/apps/photos/src/pages/_app.tsx +++ b/web/apps/photos/src/pages/_app.tsx @@ -50,11 +50,9 @@ import { createContext, useContext, useEffect, useRef, useState } from "react"; import LoadingBar from "react-top-loading-bar"; import DownloadManager from "services/download"; import { resumeExportsIfNeeded } from "services/export"; +import { isFaceIndexingEnabled } from "services/face/indexer"; import { photosLogout } from "services/logout"; -import { - getMLSearchConfig, - updateMLSearchConfig, -} from "services/machineLearning/machineLearningService"; +import { updateMLSearchConfig } from "services/machineLearning/machineLearningService"; import mlWorkManager from "services/machineLearning/mlWorkManager"; import { getFamilyPortalRedirectURL, @@ -186,9 +184,9 @@ export default function App({ Component, pageProps }: AppProps) { } const loadMlSearchState = async () => { try { - const mlSearchConfig = await getMLSearchConfig(); - setMlSearchEnabled(mlSearchConfig.enabled); - mlWorkManager.setMlSearchEnabled(mlSearchConfig.enabled); + const enabled = await isFaceIndexingEnabled(); + setMlSearchEnabled(enabled); + mlWorkManager.setMlSearchEnabled(enabled); } catch (e) { log.error("Error while loading mlSearchEnabled", e); } @@ -286,8 +284,7 @@ export default function App({ Component, pageProps }: AppProps) { const showNavBar = (show: boolean) => setShowNavBar(show); const updateMlSearchEnabled = async (enabled: boolean) => { try { - const mlSearchConfig = await getMLSearchConfig(); - mlSearchConfig.enabled = enabled; + const mlSearchConfig = { enabled }; await updateMLSearchConfig(mlSearchConfig); setMlSearchEnabled(enabled); mlWorkManager.setMlSearchEnabled(enabled); diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index 7436986144..5d5947983f 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -225,16 +225,17 @@ export const unidentifiedFaceIDs = async ( * on any client. This {@link isFaceIndexingEnabled} property, on the other * hand, denotes whether or not indexing is enabled on the current client. */ -export const isFaceIndexingEnabled = () => { +export const isFaceIndexingEnabled = async () => { if (isInternalUserForML()) { - return mlIDbStorage.getConfig( + const config = await mlIDbStorage.getConfig( ML_SEARCH_CONFIG_NAME, DEFAULT_ML_SEARCH_CONFIG, ); + return config.enabled; } // Force disabled for everyone else while we finalize it to avoid redundant // reindexing for users. - return DEFAULT_ML_SEARCH_CONFIG; + return false; }; export const setIsFaceIndexingEnabled = (enabled: boolean) => { diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index f952a3202e..8b07e72538 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -5,7 +5,6 @@ import mlIDbStorage, { ML_SEARCH_CONFIG_NAME, type MinimalPersistedFileData, } from "services/face/db-old"; -import { isFaceIndexingEnabled } from "services/face/indexer"; import { FaceIndexerWorker } from "services/face/indexer.worker"; import { getLocalFiles } from "services/fileService"; import { EnteFile } from "types/file"; @@ -24,10 +23,6 @@ export const DEFAULT_ML_SEARCH_CONFIG: MLSearchConfig = { enabled: false, }; -export async function getMLSearchConfig() { - return isFaceIndexingEnabled(); -} - export async function updateMLSearchConfig(newConfig: MLSearchConfig) { return mlIDbStorage.putConfig(ML_SEARCH_CONFIG_NAME, newConfig); } From f66170b5b22a8802146f6f2d7df9936686a328c4 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 11:53:25 +0530 Subject: [PATCH 33/77] Bypass --- web/apps/photos/src/pages/_app.tsx | 9 +++++---- web/apps/photos/src/services/face/indexer.ts | 5 ++++- .../services/machineLearning/machineLearningService.ts | 5 ----- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/web/apps/photos/src/pages/_app.tsx b/web/apps/photos/src/pages/_app.tsx index 4157634aa5..5731b292a9 100644 --- a/web/apps/photos/src/pages/_app.tsx +++ b/web/apps/photos/src/pages/_app.tsx @@ -50,9 +50,11 @@ import { createContext, useContext, useEffect, useRef, useState } from "react"; import LoadingBar from "react-top-loading-bar"; import DownloadManager from "services/download"; import { resumeExportsIfNeeded } from "services/export"; -import { isFaceIndexingEnabled } from "services/face/indexer"; +import { + isFaceIndexingEnabled, + setIsFaceIndexingEnabled, +} from "services/face/indexer"; import { photosLogout } from "services/logout"; -import { updateMLSearchConfig } from "services/machineLearning/machineLearningService"; import mlWorkManager from "services/machineLearning/mlWorkManager"; import { getFamilyPortalRedirectURL, @@ -284,8 +286,7 @@ export default function App({ Component, pageProps }: AppProps) { const showNavBar = (show: boolean) => setShowNavBar(show); const updateMlSearchEnabled = async (enabled: boolean) => { try { - const mlSearchConfig = { enabled }; - await updateMLSearchConfig(mlSearchConfig); + await setIsFaceIndexingEnabled(enabled); setMlSearchEnabled(enabled); mlWorkManager.setMlSearchEnabled(enabled); } catch (e) { diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index 5d5947983f..f931a058dc 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -238,6 +238,9 @@ export const isFaceIndexingEnabled = async () => { return false; }; -export const setIsFaceIndexingEnabled = (enabled: boolean) => { +/** + * Update the (locally stored) value of {@link isFaceIndexingEnabled}. + */ +export const setIsFaceIndexingEnabled = async (enabled: boolean) => { return mlIDbStorage.putConfig(ML_SEARCH_CONFIG_NAME, { enabled }); }; diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index 8b07e72538..4b9826da63 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -2,7 +2,6 @@ import log from "@/next/log"; import { CustomError, parseUploadErrorCodes } from "@ente/shared/error"; import PQueue from "p-queue"; import mlIDbStorage, { - ML_SEARCH_CONFIG_NAME, type MinimalPersistedFileData, } from "services/face/db-old"; import { FaceIndexerWorker } from "services/face/indexer.worker"; @@ -23,10 +22,6 @@ export const DEFAULT_ML_SEARCH_CONFIG: MLSearchConfig = { enabled: false, }; -export async function updateMLSearchConfig(newConfig: MLSearchConfig) { - return mlIDbStorage.putConfig(ML_SEARCH_CONFIG_NAME, newConfig); -} - class MLSyncContext { public token: string; public userID: number; From 3c3f9b2b48546ab2371dcc0730b1a5b9870b3b28 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 11:54:48 +0530 Subject: [PATCH 34/77] Inline --- web/apps/photos/src/services/face/db-old.ts | 9 +++++---- web/apps/photos/src/services/face/indexer.ts | 8 +++----- .../services/machineLearning/machineLearningService.ts | 8 -------- 3 files changed, 8 insertions(+), 17 deletions(-) diff --git a/web/apps/photos/src/services/face/db-old.ts b/web/apps/photos/src/services/face/db-old.ts index a70e94bee7..7dd1e71fa3 100644 --- a/web/apps/photos/src/services/face/db-old.ts +++ b/web/apps/photos/src/services/face/db-old.ts @@ -11,10 +11,7 @@ import { import isElectron from "is-electron"; import type { Person } from "services/face/people"; import type { MlFileData } from "services/face/types-old"; -import { - DEFAULT_ML_SEARCH_CONFIG, - MAX_ML_SYNC_ERROR_COUNT, -} from "services/machineLearning/machineLearningService"; +import { MAX_ML_SYNC_ERROR_COUNT } from "services/machineLearning/machineLearningService"; export interface IndexStatus { outOfSyncFilesExists: boolean; @@ -159,6 +156,10 @@ class MLIDbStorage { */ } if (oldVersion < 3) { + const DEFAULT_ML_SEARCH_CONFIG = { + enabled: false, + }; + await tx .objectStore("configs") .add(DEFAULT_ML_SEARCH_CONFIG, ML_SEARCH_CONFIG_NAME); diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index f931a058dc..5930e60c1e 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -4,7 +4,6 @@ import { wait } from "@/utils/promise"; import { type Remote } from "comlink"; import mlIDbStorage, { ML_SEARCH_CONFIG_NAME } from "services/face/db-old"; import machineLearningService, { - DEFAULT_ML_SEARCH_CONFIG, defaultMLVersion, } from "services/machineLearning/machineLearningService"; import mlWorkManager from "services/machineLearning/mlWorkManager"; @@ -227,10 +226,9 @@ export const unidentifiedFaceIDs = async ( */ export const isFaceIndexingEnabled = async () => { if (isInternalUserForML()) { - const config = await mlIDbStorage.getConfig( - ML_SEARCH_CONFIG_NAME, - DEFAULT_ML_SEARCH_CONFIG, - ); + const config = await mlIDbStorage.getConfig(ML_SEARCH_CONFIG_NAME, { + enabled: false, + }); return config.enabled; } // Force disabled for everyone else while we finalize it to avoid redundant diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index 4b9826da63..a452846623 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -14,14 +14,6 @@ const batchSize = 200; export const MAX_ML_SYNC_ERROR_COUNT = 1; -export interface MLSearchConfig { - enabled: boolean; -} - -export const DEFAULT_ML_SEARCH_CONFIG: MLSearchConfig = { - enabled: false, -}; - class MLSyncContext { public token: string; public userID: number; From c8a7152cdc6a450c7c18292da403bf6961fe41cb Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 12:01:04 +0530 Subject: [PATCH 35/77] Remove unnecessary propagation --- .../services/machineLearning/machineLearningService.ts | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index a452846623..f28c3f76c4 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -277,13 +277,8 @@ class MachineLearningService { localFile?: globalThis.File, ) { try { - const mlFileData = await this.syncFile( - enteFile, - localFile, - syncContext.userAgent, - ); + await this.syncFile(enteFile, localFile, syncContext.userAgent); syncContext.nSyncedFiles += 1; - return mlFileData; } catch (e) { log.error("ML syncFile failed", e); let error = e; @@ -320,14 +315,13 @@ class MachineLearningService { ) { const oldMlFile = await mlIDbStorage.getFile(enteFile.id); if (oldMlFile && oldMlFile.mlVersion) { - return oldMlFile; + return; } const worker = new FaceIndexerWorker(); const newMlFile = await worker.index(enteFile, localFile, userAgent); await mlIDbStorage.putFile(newMlFile); - return newMlFile; } private async persistMLFileSyncError(enteFile: EnteFile, e: Error) { From d448676b8f430dbc0b5b5193651954f0a2031bb9 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 12:09:01 +0530 Subject: [PATCH 36/77] Move --- web/apps/photos/src/services/face/indexer.ts | 70 ++++++++++++++++++- .../machineLearning/machineLearningService.ts | 51 +------------- 2 files changed, 72 insertions(+), 49 deletions(-) diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index 5930e60c1e..829638c755 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -3,6 +3,7 @@ import { ComlinkWorker } from "@/next/worker/comlink-worker"; import { wait } from "@/utils/promise"; import { type Remote } from "comlink"; import mlIDbStorage, { ML_SEARCH_CONFIG_NAME } from "services/face/db-old"; +import { getLocalFiles } from "services/fileService"; import machineLearningService, { defaultMLVersion, } from "services/machineLearning/machineLearningService"; @@ -10,7 +11,7 @@ import mlWorkManager from "services/machineLearning/mlWorkManager"; import type { EnteFile } from "types/file"; import { isInternalUserForML } from "utils/user"; import { indexableAndIndexedCounts, markIndexingFailed } from "./db"; -import type { IndexStatus } from "./db-old"; +import type { IndexStatus, MinimalPersistedFileData } from "./db-old"; import { indexFaces } from "./f-index"; import { FaceIndexerWorker } from "./indexer.worker"; @@ -242,3 +243,70 @@ export const isFaceIndexingEnabled = async () => { export const setIsFaceIndexingEnabled = async (enabled: boolean) => { return mlIDbStorage.putConfig(ML_SEARCH_CONFIG_NAME, { enabled }); }; + +export const syncLocalFiles = async (userID: number) => { + const startTime = Date.now(); + const localFilesMap = await getLocalFilesMap(userID); + + const db = await mlIDbStorage.db; + const tx = db.transaction("files", "readwrite"); + const mlFileIdsArr = await mlIDbStorage.getAllFileIdsForUpdate(tx); + const mlFileIds = new Set(); + mlFileIdsArr.forEach((mlFileId) => mlFileIds.add(mlFileId)); + + const newFileIds: Array = []; + for (const localFileId of localFilesMap.keys()) { + if (!mlFileIds.has(localFileId)) { + newFileIds.push(localFileId); + } + } + + let updated = false; + if (newFileIds.length > 0) { + log.info("newFiles: ", newFileIds.length); + const newFiles = newFileIds.map( + (fileId) => + ({ + fileId, + mlVersion: 0, + errorCount: 0, + }) as MinimalPersistedFileData, + ); + await mlIDbStorage.putAllFiles(newFiles, tx); + updated = true; + } + + const removedFileIds: Array = []; + for (const mlFileId of mlFileIds) { + if (!localFilesMap.has(mlFileId)) { + removedFileIds.push(mlFileId); + } + } + + if (removedFileIds.length > 0) { + log.info("removedFiles: ", removedFileIds.length); + await mlIDbStorage.removeAllFiles(removedFileIds, tx); + updated = true; + } + + await tx.done; + + if (updated) { + // TODO: should do in same transaction + await mlIDbStorage.incrementIndexVersion("files"); + } + + log.info("syncLocalFiles", Date.now() - startTime, "ms"); + + return localFilesMap; +}; + +const getLocalFilesMap = async (userID: number) => { + const localFiles = await getLocalFiles(); + + const personalFiles = localFiles.filter((f) => f.ownerID === userID); + const localFilesMap = new Map(); + personalFiles.forEach((f) => localFilesMap.set(f.id, f)); + + return localFilesMap; +}; diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index f28c3f76c4..4aa2235db4 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -4,6 +4,7 @@ import PQueue from "p-queue"; import mlIDbStorage, { type MinimalPersistedFileData, } from "services/face/db-old"; +import { syncLocalFiles } from "services/face/indexer"; import { FaceIndexerWorker } from "services/face/indexer.worker"; import { getLocalFiles } from "services/fileService"; import { EnteFile } from "types/file"; @@ -65,7 +66,8 @@ class MachineLearningService { const syncContext = await this.getSyncContext(token, userID, userAgent); - await this.syncLocalFiles(syncContext); + const localFiles = await syncLocalFiles(userID); + syncContext.localFilesMap = localFiles; await this.getOutOfSyncFiles(syncContext); @@ -102,53 +104,6 @@ class MachineLearningService { return syncContext.localFilesMap; } - private async syncLocalFiles(syncContext: MLSyncContext) { - const startTime = Date.now(); - const localFilesMap = await this.getLocalFilesMap(syncContext); - - const db = await mlIDbStorage.db; - const tx = db.transaction("files", "readwrite"); - const mlFileIdsArr = await mlIDbStorage.getAllFileIdsForUpdate(tx); - const mlFileIds = new Set(); - mlFileIdsArr.forEach((mlFileId) => mlFileIds.add(mlFileId)); - - const newFileIds: Array = []; - for (const localFileId of localFilesMap.keys()) { - if (!mlFileIds.has(localFileId)) { - newFileIds.push(localFileId); - } - } - - let updated = false; - if (newFileIds.length > 0) { - log.info("newFiles: ", newFileIds.length); - const newFiles = newFileIds.map((fileId) => this.newMlData(fileId)); - await mlIDbStorage.putAllFiles(newFiles, tx); - updated = true; - } - - const removedFileIds: Array = []; - for (const mlFileId of mlFileIds) { - if (!localFilesMap.has(mlFileId)) { - removedFileIds.push(mlFileId); - } - } - - if (removedFileIds.length > 0) { - log.info("removedFiles: ", removedFileIds.length); - await mlIDbStorage.removeAllFiles(removedFileIds, tx); - updated = true; - } - - await tx.done; - - if (updated) { - // TODO: should do in same transaction - await mlIDbStorage.incrementIndexVersion("files"); - } - - log.info("syncLocalFiles", Date.now() - startTime, "ms"); - } private async getOutOfSyncFiles(syncContext: MLSyncContext) { const startTime = Date.now(); From b17933a2b3f906c7cf4cd424525758ac2db6f284 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 12:14:02 +0530 Subject: [PATCH 37/77] Tweak --- web/apps/photos/src/services/face/indexer.ts | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index 829638c755..98a22b3e1e 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -246,7 +246,7 @@ export const setIsFaceIndexingEnabled = async (enabled: boolean) => { export const syncLocalFiles = async (userID: number) => { const startTime = Date.now(); - const localFilesMap = await getLocalFilesMap(userID); + const localFilesMap = await localUserOwnedFilesByID(userID); const db = await mlIDbStorage.db; const tx = db.transaction("files", "readwrite"); @@ -301,12 +301,18 @@ export const syncLocalFiles = async (userID: number) => { return localFilesMap; }; -const getLocalFilesMap = async (userID: number) => { +/** + * Return a map of all {@link EnteFile}s owned by {@link userID} that we know + * about locally, indexed by their {@link fileID}. + * + * @param userID Restrict the returned files to those owned by a {@link userID}. + */ +const localUserOwnedFilesByID = async ( + userID: number, +): Promise> => { + const result = new Map(); const localFiles = await getLocalFiles(); - const personalFiles = localFiles.filter((f) => f.ownerID === userID); - const localFilesMap = new Map(); - personalFiles.forEach((f) => localFilesMap.set(f.id, f)); - - return localFilesMap; + personalFiles.forEach((f) => result.set(f.id, f)); + return result; }; From 6b1484671bf90f9e82b256c30001e74a43e43b87 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 12:24:34 +0530 Subject: [PATCH 38/77] Add remove --- web/apps/photos/src/services/face/db.ts | 26 +++++++++++++++++--- web/apps/photos/src/services/face/indexer.ts | 6 ++--- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/web/apps/photos/src/services/face/db.ts b/web/apps/photos/src/services/face/db.ts index 85bb322056..1625376a79 100644 --- a/web/apps/photos/src/services/face/db.ts +++ b/web/apps/photos/src/services/face/db.ts @@ -197,18 +197,36 @@ export const addFileEntry = async (fileID: number) => { return tx.done; }; +/** + * Remove a file's entry and face index (if any) from the face DB. + * + * @param fileID The ID of an {@link EnteFile} + * + * This function is invoked when the user has deleted a file, and we want to + * also prune that file's data from the face DB. + */ +export const removeFile = async (fileID: number) => { + const db = await faceDB(); + const tx = db.transaction(["face-index", "file-status"], "readwrite"); + return Promise.all([ + tx.objectStore("face-index").delete(fileID), + tx.objectStore("file-status").delete(fileID), + tx.done, + ]); +}; + /** * Return the count of files that can be, and that have been, indexed. */ -export const indexableAndIndexedCounts = async () => { +export const indexedAndIndexableCounts = async () => { const db = await faceDB(); - const tx = db.transaction(["file-status", "face-index"], "readonly"); + const tx = db.transaction(["face-index", "file-status"], "readwrite"); + const indexedCount = await tx.objectStore("face-index").count(); const indexableCount = await tx .objectStore("file-status") .index("isIndexable") .count(IDBKeyRange.only(1)); - const indexedCount = await tx.objectStore("face-index").count(); - return { indexableCount, indexedCount }; + return { indexedCount, indexableCount }; }; /** diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index 98a22b3e1e..a10ff34bb1 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -10,7 +10,7 @@ import machineLearningService, { import mlWorkManager from "services/machineLearning/mlWorkManager"; import type { EnteFile } from "types/file"; import { isInternalUserForML } from "utils/user"; -import { indexableAndIndexedCounts, markIndexingFailed } from "./db"; +import { indexedAndIndexableCounts, markIndexingFailed } from "./db"; import type { IndexStatus, MinimalPersistedFileData } from "./db-old"; import { indexFaces } from "./f-index"; import { FaceIndexerWorker } from "./indexer.worker"; @@ -161,7 +161,7 @@ export interface FaceIndexingStatus { export const faceIndexingStatus = async (): Promise => { const isSyncing = machineLearningService.isSyncing; - const { indexableCount, indexedCount } = await indexableAndIndexedCounts(); + const { indexedCount, indexableCount } = await indexedAndIndexableCounts(); let phase: FaceIndexingStatus["phase"]; if (indexedCount < indexableCount) { @@ -176,8 +176,8 @@ export const faceIndexingStatus = async (): Promise => { const indexingStatus = { phase, - nTotalFiles: indexableCount, nSyncedFiles: indexedCount, + nTotalFiles: indexableCount, }; const indexStatus0 = await mlIDbStorage.getIndexStatus(defaultMLVersion); From 53dea9dcf3f6334d2635b677bc637f2b12bf0615 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 12:50:14 +0530 Subject: [PATCH 39/77] Sync --- web/apps/photos/src/services/face/db.ts | 46 +++++++++++++++++++------ 1 file changed, 36 insertions(+), 10 deletions(-) diff --git a/web/apps/photos/src/services/face/db.ts b/web/apps/photos/src/services/face/db.ts index 1625376a79..bfa54e7d6a 100644 --- a/web/apps/photos/src/services/face/db.ts +++ b/web/apps/photos/src/services/face/db.ts @@ -198,21 +198,47 @@ export const addFileEntry = async (fileID: number) => { }; /** - * Remove a file's entry and face index (if any) from the face DB. + * Sync entries in the face DB to match the given list of local file IDs. * - * @param fileID The ID of an {@link EnteFile} + * @param localFileIDs The IDs of all the files that the client is aware of, + * filtered to only keep the files that the user owns. * - * This function is invoked when the user has deleted a file, and we want to - * also prune that file's data from the face DB. + * This function syncs the state of file entries in face DB to the state of file + * entries stored otherwise by the local client. + * + * - Files (identified by their ID) that are present locally but are not yet in + * face DB get a fresh entry in face DB (and are marked as indexable). + * + * - Files that are not present locally but still exist in face DB are removed + * from face DB (including its face index, if any). */ -export const removeFile = async (fileID: number) => { +export const syncWithLocalUserOwnedFileIDs = async (localFileIDs: number[]) => { const db = await faceDB(); const tx = db.transaction(["face-index", "file-status"], "readwrite"); - return Promise.all([ - tx.objectStore("face-index").delete(fileID), - tx.objectStore("file-status").delete(fileID), - tx.done, - ]); + const fdbFileIDs = await tx.objectStore("file-status").getAllKeys(); + + const local = new Set(localFileIDs); + const fdb = new Set(fdbFileIDs); + + const newFileIDs = localFileIDs.filter((id) => !fdb.has(id)); + const removedFileIDs = fdbFileIDs.filter((id) => !local.has(id)); + + return Promise.all( + [ + newFileIDs.map((id) => + tx.objectStore("file-status").put({ + fileID: id, + isIndexable: 1, + failureCount: 0, + }), + ), + removedFileIDs.map((id) => + tx.objectStore("file-status").delete(id), + ), + removedFileIDs.map((id) => tx.objectStore("face-index").delete(id)), + tx.done, + ].flat(), + ); }; /** From 6e6c88826e95f96888071525d4b77a0104f2f8d6 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 13:01:11 +0530 Subject: [PATCH 40/77] t --- web/apps/photos/src/services/face/indexer.ts | 5 ++--- .../src/services/machineLearning/machineLearningService.ts | 1 - 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index a10ff34bb1..9f3cc4a9d3 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -245,9 +245,10 @@ export const setIsFaceIndexingEnabled = async (enabled: boolean) => { }; export const syncLocalFiles = async (userID: number) => { - const startTime = Date.now(); const localFilesMap = await localUserOwnedFilesByID(userID); + // const localFileIDs = new Set(localFilesMap.keys()); + const db = await mlIDbStorage.db; const tx = db.transaction("files", "readwrite"); const mlFileIdsArr = await mlIDbStorage.getAllFileIdsForUpdate(tx); @@ -296,8 +297,6 @@ export const syncLocalFiles = async (userID: number) => { await mlIDbStorage.incrementIndexVersion("files"); } - log.info("syncLocalFiles", Date.now() - startTime, "ms"); - return localFilesMap; }; diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index 4aa2235db4..f1de281efa 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -104,7 +104,6 @@ class MachineLearningService { return syncContext.localFilesMap; } - private async getOutOfSyncFiles(syncContext: MLSyncContext) { const startTime = Date.now(); const fileIds = await mlIDbStorage.getFileIds( From 23c73a83eb9dbda5fafe9ecf430ba4bf9b739181 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 13:16:15 +0530 Subject: [PATCH 41/77] Inline --- web/apps/photos/src/services/face/f-index.ts | 51 +++++++++++++++----- web/apps/photos/src/services/face/file.ts | 37 -------------- web/apps/photos/src/services/face/people.ts | 4 ++ 3 files changed, 44 insertions(+), 48 deletions(-) delete mode 100644 web/apps/photos/src/services/face/file.ts diff --git a/web/apps/photos/src/services/face/f-index.ts b/web/apps/photos/src/services/face/f-index.ts index 714f89c09f..9f871d4367 100644 --- a/web/apps/photos/src/services/face/f-index.ts +++ b/web/apps/photos/src/services/face/f-index.ts @@ -1,7 +1,9 @@ import { FILE_TYPE } from "@/media/file-type"; +import { decodeLivePhoto } from "@/media/live-photo"; import log from "@/next/log"; import { workerBridge } from "@/next/worker/worker-bridge"; import { Matrix } from "ml-matrix"; +import DownloadManager from "services/download"; import { defaultMLVersion } from "services/machineLearning/machineLearningService"; import { getSimilarityTransformation } from "similarity-transformation"; import { @@ -12,9 +14,8 @@ import { translate, } from "transformation-matrix"; import type { EnteFile } from "types/file"; -import { logIdentifier } from "utils/file"; +import { getRenderableImage, logIdentifier } from "utils/file"; import { saveFaceCrop } from "./crop"; -import { fetchImageBitmap, getLocalFileImageBitmap } from "./file"; import { clamp, grayscaleIntMatrixFromNormalized2List, @@ -39,11 +40,20 @@ import type { Face, FaceDetection, MlFileData } from "./types-old"; * they can be saved locally for offline use, and encrypts and uploads them to * the user's remote storage so that their other devices can download them * instead of needing to reindex. + * + * @param enteFile The {@link EnteFile} to index. + * + * @param file The contents of {@link enteFile} as a web {@link File}, if + * available. These are used when they are provided, otherwise the file is + * downloaded and decrypted from remote. */ -export const indexFaces = async (enteFile: EnteFile, localFile?: File) => { +export const indexFaces = async ( + enteFile: EnteFile, + file: File | undefined, +) => { const startTime = Date.now(); - const imageBitmap = await fetchOrCreateImageBitmap(enteFile, localFile); + const imageBitmap = await fetchOrCreateImageBitmap(enteFile, file); let mlFile: MlFileData; try { mlFile = await indexFaces_(enteFile, imageBitmap); @@ -60,20 +70,17 @@ export const indexFaces = async (enteFile: EnteFile, localFile?: File) => { }; /** - * Return a {@link ImageBitmap}, using {@link localFile} if present otherwise + * Return a {@link ImageBitmap}, using {@link file} if present otherwise * downloading the source image corresponding to {@link enteFile} from remote. */ -const fetchOrCreateImageBitmap = async ( - enteFile: EnteFile, - localFile: File, -) => { +const fetchOrCreateImageBitmap = async (enteFile: EnteFile, file: File) => { const fileType = enteFile.metadata.fileType; - if (localFile) { + if (file) { // TODO-ML(MR): Could also be image part of live photo? if (fileType !== FILE_TYPE.IMAGE) throw new Error("Local file of only image type is supported"); - return await getLocalFileImageBitmap(enteFile, localFile); + return await getLocalFileImageBitmap(enteFile, file); } else if ([FILE_TYPE.IMAGE, FILE_TYPE.LIVE_PHOTO].includes(fileType)) { return await fetchImageBitmap(enteFile); } else { @@ -81,6 +88,28 @@ const fetchOrCreateImageBitmap = async ( } }; +const fetchImageBitmap = async (enteFile: EnteFile) => + fetchRenderableBlob(enteFile).then(createImageBitmap); + +const fetchRenderableBlob = async (enteFile: EnteFile) => { + const fileStream = await DownloadManager.getFile(enteFile); + const fileBlob = await new Response(fileStream).blob(); + if (enteFile.metadata.fileType === FILE_TYPE.IMAGE) { + return getRenderableImage(enteFile.metadata.title, fileBlob); + } else { + const { imageFileName, imageData } = await decodeLivePhoto( + enteFile.metadata.title, + fileBlob, + ); + return getRenderableImage(imageFileName, new Blob([imageData])); + } +}; + +const getLocalFileImageBitmap = async (enteFile: EnteFile, localFile: File) => + createImageBitmap( + await getRenderableImage(enteFile.metadata.title, localFile), + ); + const indexFaces_ = async (enteFile: EnteFile, imageBitmap: ImageBitmap) => { const fileID = enteFile.id; const { width, height } = imageBitmap; diff --git a/web/apps/photos/src/services/face/file.ts b/web/apps/photos/src/services/face/file.ts deleted file mode 100644 index b482af3fb5..0000000000 --- a/web/apps/photos/src/services/face/file.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { FILE_TYPE } from "@/media/file-type"; -import { decodeLivePhoto } from "@/media/live-photo"; -import DownloadManager from "services/download"; -import { getLocalFiles } from "services/fileService"; -import { EnteFile } from "types/file"; -import { getRenderableImage } from "utils/file"; - -export async function getLocalFile(fileId: number) { - const localFiles = await getLocalFiles(); - return localFiles.find((f) => f.id === fileId); -} - -export const fetchImageBitmap = async (file: EnteFile) => - fetchRenderableBlob(file).then(createImageBitmap); - -async function fetchRenderableBlob(file: EnteFile) { - const fileStream = await DownloadManager.getFile(file); - const fileBlob = await new Response(fileStream).blob(); - if (file.metadata.fileType === FILE_TYPE.IMAGE) { - return await getRenderableImage(file.metadata.title, fileBlob); - } else { - const { imageFileName, imageData } = await decodeLivePhoto( - file.metadata.title, - fileBlob, - ); - return await getRenderableImage(imageFileName, new Blob([imageData])); - } -} - -export async function getLocalFileImageBitmap( - enteFile: EnteFile, - localFile: globalThis.File, -) { - let fileBlob = localFile as Blob; - fileBlob = await getRenderableImage(enteFile.metadata.title, fileBlob); - return createImageBitmap(fileBlob); -} diff --git a/web/apps/photos/src/services/face/people.ts b/web/apps/photos/src/services/face/people.ts index d118cb4f90..311183768c 100644 --- a/web/apps/photos/src/services/face/people.ts +++ b/web/apps/photos/src/services/face/people.ts @@ -84,6 +84,10 @@ export const syncPeopleIndex = async () => { : best, ); +export async function getLocalFile(fileId: number) { + const localFiles = await getLocalFiles(); + return localFiles.find((f) => f.id === fileId); +} if (personFace && !personFace.crop?.cacheKey) { const file = await getLocalFile(personFace.fileId); From fd4a7889532d33a1005ee32fa8afe2d2751d0dc8 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 13:22:46 +0530 Subject: [PATCH 42/77] Checked that the image part is passed as the file --- web/apps/photos/src/services/face/f-index.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/web/apps/photos/src/services/face/f-index.ts b/web/apps/photos/src/services/face/f-index.ts index 9f871d4367..4ed6c4792f 100644 --- a/web/apps/photos/src/services/face/f-index.ts +++ b/web/apps/photos/src/services/face/f-index.ts @@ -76,10 +76,6 @@ export const indexFaces = async ( const fetchOrCreateImageBitmap = async (enteFile: EnteFile, file: File) => { const fileType = enteFile.metadata.fileType; if (file) { - // TODO-ML(MR): Could also be image part of live photo? - if (fileType !== FILE_TYPE.IMAGE) - throw new Error("Local file of only image type is supported"); - return await getLocalFileImageBitmap(enteFile, file); } else if ([FILE_TYPE.IMAGE, FILE_TYPE.LIVE_PHOTO].includes(fileType)) { return await fetchImageBitmap(enteFile); From c71e56ec43bb0ac0e2a48876aaa2e8b7394c8dca Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 13:26:09 +0530 Subject: [PATCH 43/77] inl --- web/apps/photos/src/services/face/f-index.ts | 21 ++++++++------------ 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/web/apps/photos/src/services/face/f-index.ts b/web/apps/photos/src/services/face/f-index.ts index 4ed6c4792f..c47d8a677c 100644 --- a/web/apps/photos/src/services/face/f-index.ts +++ b/web/apps/photos/src/services/face/f-index.ts @@ -53,7 +53,7 @@ export const indexFaces = async ( ) => { const startTime = Date.now(); - const imageBitmap = await fetchOrCreateImageBitmap(enteFile, file); + const imageBitmap = await fetchAndCreateImageBitmap(enteFile, file); let mlFile: MlFileData; try { mlFile = await indexFaces_(enteFile, imageBitmap); @@ -73,31 +73,26 @@ export const indexFaces = async ( * Return a {@link ImageBitmap}, using {@link file} if present otherwise * downloading the source image corresponding to {@link enteFile} from remote. */ -const fetchOrCreateImageBitmap = async (enteFile: EnteFile, file: File) => { - const fileType = enteFile.metadata.fileType; - if (file) { - return await getLocalFileImageBitmap(enteFile, file); - } else if ([FILE_TYPE.IMAGE, FILE_TYPE.LIVE_PHOTO].includes(fileType)) { - return await fetchImageBitmap(enteFile); - } else { - throw new Error(`Cannot index unsupported file type ${fileType}`); - } -}; +const fetchAndCreateImageBitmap = async (enteFile: EnteFile, file: File) => + file ? getLocalFileImageBitmap(enteFile, file) : fetchImageBitmap(enteFile); const fetchImageBitmap = async (enteFile: EnteFile) => fetchRenderableBlob(enteFile).then(createImageBitmap); const fetchRenderableBlob = async (enteFile: EnteFile) => { + const fileType = enteFile.metadata.fileType; const fileStream = await DownloadManager.getFile(enteFile); const fileBlob = await new Response(fileStream).blob(); - if (enteFile.metadata.fileType === FILE_TYPE.IMAGE) { + if (enteFile.metadata.fileType == FILE_TYPE.IMAGE) { return getRenderableImage(enteFile.metadata.title, fileBlob); - } else { + } else if (enteFile.metadata.fileType == FILE_TYPE.LIVE_PHOTO) { const { imageFileName, imageData } = await decodeLivePhoto( enteFile.metadata.title, fileBlob, ); return getRenderableImage(imageFileName, new Blob([imageData])); + } else { + throw new Error(`Cannot index unsupported file type ${fileType}`); } }; From 3fc41aecca308cf34c9e95471278b186310325de Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 13:32:10 +0530 Subject: [PATCH 44/77] inl --- web/apps/photos/src/services/face/f-index.ts | 26 +++++++++----------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/web/apps/photos/src/services/face/f-index.ts b/web/apps/photos/src/services/face/f-index.ts index c47d8a677c..d2c5579a2f 100644 --- a/web/apps/photos/src/services/face/f-index.ts +++ b/web/apps/photos/src/services/face/f-index.ts @@ -53,7 +53,9 @@ export const indexFaces = async ( ) => { const startTime = Date.now(); - const imageBitmap = await fetchAndCreateImageBitmap(enteFile, file); + const imageBitmap = await renderableImageBlob(enteFile, file).then( + createImageBitmap, + ); let mlFile: MlFileData; try { mlFile = await indexFaces_(enteFile, imageBitmap); @@ -70,22 +72,21 @@ export const indexFaces = async ( }; /** - * Return a {@link ImageBitmap}, using {@link file} if present otherwise + * Return a "renderable" image blob, using {@link file} if present otherwise * downloading the source image corresponding to {@link enteFile} from remote. */ -const fetchAndCreateImageBitmap = async (enteFile: EnteFile, file: File) => - file ? getLocalFileImageBitmap(enteFile, file) : fetchImageBitmap(enteFile); - -const fetchImageBitmap = async (enteFile: EnteFile) => - fetchRenderableBlob(enteFile).then(createImageBitmap); +const renderableImageBlob = async (enteFile: EnteFile, file: File) => + file + ? getRenderableImage(enteFile.metadata.title, file) + : fetchRenderableBlob(enteFile); const fetchRenderableBlob = async (enteFile: EnteFile) => { - const fileType = enteFile.metadata.fileType; const fileStream = await DownloadManager.getFile(enteFile); const fileBlob = await new Response(fileStream).blob(); - if (enteFile.metadata.fileType == FILE_TYPE.IMAGE) { + const fileType = enteFile.metadata.fileType; + if (fileType == FILE_TYPE.IMAGE) { return getRenderableImage(enteFile.metadata.title, fileBlob); - } else if (enteFile.metadata.fileType == FILE_TYPE.LIVE_PHOTO) { + } else if (fileType == FILE_TYPE.LIVE_PHOTO) { const { imageFileName, imageData } = await decodeLivePhoto( enteFile.metadata.title, fileBlob, @@ -96,11 +97,6 @@ const fetchRenderableBlob = async (enteFile: EnteFile) => { } }; -const getLocalFileImageBitmap = async (enteFile: EnteFile, localFile: File) => - createImageBitmap( - await getRenderableImage(enteFile.metadata.title, localFile), - ); - const indexFaces_ = async (enteFile: EnteFile, imageBitmap: ImageBitmap) => { const fileID = enteFile.id; const { width, height } = imageBitmap; From ac8677d7b4f059220c6ec3f017919b2c9cb2aef6 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 13:35:38 +0530 Subject: [PATCH 45/77] Filter instead of marking as errors --- web/apps/photos/src/services/face/db.ts | 8 +++++--- web/apps/photos/src/services/face/f-index.ts | 1 + web/apps/photos/src/services/face/indexer.ts | 14 ++++++++++---- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/web/apps/photos/src/services/face/db.ts b/web/apps/photos/src/services/face/db.ts index bfa54e7d6a..fe5a93cdb2 100644 --- a/web/apps/photos/src/services/face/db.ts +++ b/web/apps/photos/src/services/face/db.ts @@ -198,10 +198,12 @@ export const addFileEntry = async (fileID: number) => { }; /** - * Sync entries in the face DB to match the given list of local file IDs. + * Sync entries in the face DB to align with the given list of local indexable + * file IDs. * * @param localFileIDs The IDs of all the files that the client is aware of, - * filtered to only keep the files that the user owns. + * filtered to only keep the files that the user owns and the formats that can + * be indexed by our current face indexing pipeline. * * This function syncs the state of file entries in face DB to the state of file * entries stored otherwise by the local client. @@ -212,7 +214,7 @@ export const addFileEntry = async (fileID: number) => { * - Files that are not present locally but still exist in face DB are removed * from face DB (including its face index, if any). */ -export const syncWithLocalUserOwnedFileIDs = async (localFileIDs: number[]) => { +export const syncWithLocalIndexableFileIDs = async (localFileIDs: number[]) => { const db = await faceDB(); const tx = db.transaction(["face-index", "file-status"], "readwrite"); const fdbFileIDs = await tx.objectStore("file-status").getAllKeys(); diff --git a/web/apps/photos/src/services/face/f-index.ts b/web/apps/photos/src/services/face/f-index.ts index d2c5579a2f..fa9fdea86a 100644 --- a/web/apps/photos/src/services/face/f-index.ts +++ b/web/apps/photos/src/services/face/f-index.ts @@ -93,6 +93,7 @@ const fetchRenderableBlob = async (enteFile: EnteFile) => { ); return getRenderableImage(imageFileName, new Blob([imageData])); } else { + // A layer above us should've already filtered these out. throw new Error(`Cannot index unsupported file type ${fileType}`); } }; diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index 9f3cc4a9d3..c880de2ef2 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -1,3 +1,4 @@ +import { FILE_TYPE } from "@/media/file-type"; import log from "@/next/log"; import { ComlinkWorker } from "@/next/worker/comlink-worker"; import { wait } from "@/utils/promise"; @@ -245,7 +246,7 @@ export const setIsFaceIndexingEnabled = async (enabled: boolean) => { }; export const syncLocalFiles = async (userID: number) => { - const localFilesMap = await localUserOwnedFilesByID(userID); + const localFilesMap = await localIndexableFilesByID(userID); // const localFileIDs = new Set(localFilesMap.keys()); @@ -306,12 +307,17 @@ export const syncLocalFiles = async (userID: number) => { * * @param userID Restrict the returned files to those owned by a {@link userID}. */ -const localUserOwnedFilesByID = async ( +const localIndexableFilesByID = async ( userID: number, ): Promise> => { const result = new Map(); const localFiles = await getLocalFiles(); - const personalFiles = localFiles.filter((f) => f.ownerID === userID); - personalFiles.forEach((f) => result.set(f.id, f)); + const indexableTypes = [FILE_TYPE.IMAGE, FILE_TYPE.LIVE_PHOTO]; + const indexableFiles = localFiles.filter( + (f) => + f.ownerID == userID && indexableTypes.includes(f.metadata.fileType), + ); + + indexableFiles.forEach((f) => result.set(f.id, f)); return result; }; From 4b202d2dda4d698676ea53bfc9d3182cfeebb2ec Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 13:45:23 +0530 Subject: [PATCH 46/77] r --- web/apps/photos/src/services/face/f-index.ts | 2 +- web/apps/photos/src/services/face/remote.ts | 2 +- web/apps/photos/src/services/face/types-old.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/web/apps/photos/src/services/face/f-index.ts b/web/apps/photos/src/services/face/f-index.ts index fa9fdea86a..a73e492bb9 100644 --- a/web/apps/photos/src/services/face/f-index.ts +++ b/web/apps/photos/src/services/face/f-index.ts @@ -103,7 +103,7 @@ const indexFaces_ = async (enteFile: EnteFile, imageBitmap: ImageBitmap) => { const { width, height } = imageBitmap; const imageDimensions = { width, height }; const mlFile: MlFileData = { - fileId: fileID, + fileID: fileID, mlVersion: defaultMLVersion, imageDimensions, errorCount: 0, diff --git a/web/apps/photos/src/services/face/remote.ts b/web/apps/photos/src/services/face/remote.ts index 36ef724bf3..26f57125af 100644 --- a/web/apps/photos/src/services/face/remote.ts +++ b/web/apps/photos/src/services/face/remote.ts @@ -141,7 +141,7 @@ function LocalFileMlDataToServerFileMl( } const faceEmbeddings = new ServerFaceEmbeddings(faces, userAgent, 1); return new ServerFileMl( - localFileMlData.fileId, + localFileMlData.fileID, faceEmbeddings, imageDimensions.height, imageDimensions.width, diff --git a/web/apps/photos/src/services/face/types-old.ts b/web/apps/photos/src/services/face/types-old.ts index ad6ef68743..5135881eb8 100644 --- a/web/apps/photos/src/services/face/types-old.ts +++ b/web/apps/photos/src/services/face/types-old.ts @@ -17,7 +17,7 @@ export interface Face { } export interface MlFileData { - fileId: number; + fileID: number; faces?: Face[]; imageDimensions?: Dimensions; mlVersion: number; From 6d3391528da004e4e63ae5de930ac06a1dea1e59 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 13:48:36 +0530 Subject: [PATCH 47/77] Closer --- web/apps/photos/src/services/face/f-index.ts | 3 ++- web/apps/photos/src/services/face/remote.ts | 5 ++--- web/apps/photos/src/services/face/types-old.ts | 5 +++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/web/apps/photos/src/services/face/f-index.ts b/web/apps/photos/src/services/face/f-index.ts index a73e492bb9..d64a6d1883 100644 --- a/web/apps/photos/src/services/face/f-index.ts +++ b/web/apps/photos/src/services/face/f-index.ts @@ -105,7 +105,8 @@ const indexFaces_ = async (enteFile: EnteFile, imageBitmap: ImageBitmap) => { const mlFile: MlFileData = { fileID: fileID, mlVersion: defaultMLVersion, - imageDimensions, + width, + height, errorCount: 0, }; diff --git a/web/apps/photos/src/services/face/remote.ts b/web/apps/photos/src/services/face/remote.ts index 26f57125af..f0185a44e2 100644 --- a/web/apps/photos/src/services/face/remote.ts +++ b/web/apps/photos/src/services/face/remote.ts @@ -116,7 +116,6 @@ function LocalFileMlDataToServerFileMl( if (localFileMlData.errorCount > 0) { return null; } - const imageDimensions = localFileMlData.imageDimensions; const faces: ServerFace[] = []; for (let i = 0; i < localFileMlData.faces.length; i++) { @@ -143,7 +142,7 @@ function LocalFileMlDataToServerFileMl( return new ServerFileMl( localFileMlData.fileID, faceEmbeddings, - imageDimensions.height, - imageDimensions.width, + localFileMlData.height, + localFileMlData.width, ); } diff --git a/web/apps/photos/src/services/face/types-old.ts b/web/apps/photos/src/services/face/types-old.ts index 5135881eb8..c719c2e3dd 100644 --- a/web/apps/photos/src/services/face/types-old.ts +++ b/web/apps/photos/src/services/face/types-old.ts @@ -1,4 +1,4 @@ -import type { Box, Dimensions, Point } from "./types"; +import type { Box, Point } from "./types"; export interface FaceDetection { // box and landmarks is relative to image dimentions stored at mlFileData @@ -19,7 +19,8 @@ export interface Face { export interface MlFileData { fileID: number; faces?: Face[]; - imageDimensions?: Dimensions; + width: number; + height: number; mlVersion: number; errorCount: number; } From 91be44c4c501655c5e759e21cc05fc646c17d95d Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 13:53:18 +0530 Subject: [PATCH 48/77] dup --- web/apps/photos/src/services/face/db-old.ts | 2 +- web/apps/photos/src/services/face/f-index.ts | 5 ++++- web/apps/photos/src/services/face/indexer.ts | 4 ++-- web/apps/photos/src/services/face/types-old.ts | 3 +++ .../src/services/machineLearning/machineLearningService.ts | 4 ++-- 5 files changed, 12 insertions(+), 6 deletions(-) diff --git a/web/apps/photos/src/services/face/db-old.ts b/web/apps/photos/src/services/face/db-old.ts index 7dd1e71fa3..7aabe57ac2 100644 --- a/web/apps/photos/src/services/face/db-old.ts +++ b/web/apps/photos/src/services/face/db-old.ts @@ -27,7 +27,7 @@ export interface IndexStatus { * server ML data shape here exactly. */ export interface MinimalPersistedFileData { - fileId: number; + fileID: number; mlVersion: number; errorCount: number; faces?: { personId?: number; id: string }[]; diff --git a/web/apps/photos/src/services/face/f-index.ts b/web/apps/photos/src/services/face/f-index.ts index d64a6d1883..ac7afa5d7d 100644 --- a/web/apps/photos/src/services/face/f-index.ts +++ b/web/apps/photos/src/services/face/f-index.ts @@ -104,9 +104,12 @@ const indexFaces_ = async (enteFile: EnteFile, imageBitmap: ImageBitmap) => { const imageDimensions = { width, height }; const mlFile: MlFileData = { fileID: fileID, - mlVersion: defaultMLVersion, width, height, + faceEmbedding: { + version: 1, + }, + mlVersion: defaultMLVersion, errorCount: 0, }; diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index c880de2ef2..f7f3a7ba43 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -267,9 +267,9 @@ export const syncLocalFiles = async (userID: number) => { if (newFileIds.length > 0) { log.info("newFiles: ", newFileIds.length); const newFiles = newFileIds.map( - (fileId) => + (fileID) => ({ - fileId, + fileID, mlVersion: 0, errorCount: 0, }) as MinimalPersistedFileData, diff --git a/web/apps/photos/src/services/face/types-old.ts b/web/apps/photos/src/services/face/types-old.ts index c719c2e3dd..d81ebda5b5 100644 --- a/web/apps/photos/src/services/face/types-old.ts +++ b/web/apps/photos/src/services/face/types-old.ts @@ -21,6 +21,9 @@ export interface MlFileData { faces?: Face[]; width: number; height: number; + faceEmbedding: { + version: number; + }; mlVersion: number; errorCount: number; } diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index f1de281efa..654358712e 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -80,9 +80,9 @@ class MachineLearningService { return !error && nOutOfSyncFiles > 0; } - private newMlData(fileId: number) { + private newMlData(fileID: number) { return { - fileId, + fileID, mlVersion: 0, errorCount: 0, } as MinimalPersistedFileData; From 40d35e157edecca13a946e209dd53c9c7e704af1 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 13:54:50 +0530 Subject: [PATCH 49/77] t --- web/apps/photos/src/services/face/remote.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/web/apps/photos/src/services/face/remote.ts b/web/apps/photos/src/services/face/remote.ts index f0185a44e2..791534cac0 100644 --- a/web/apps/photos/src/services/face/remote.ts +++ b/web/apps/photos/src/services/face/remote.ts @@ -36,11 +36,11 @@ class ServerFileMl { public fileID: number; public height?: number; public width?: number; - public faceEmbedding: ServerFaceEmbeddings; + public faceEmbedding: ServerFaceEmbedding; public constructor( fileID: number, - faceEmbedding: ServerFaceEmbeddings, + faceEmbedding: ServerFaceEmbedding, height?: number, width?: number, ) { @@ -51,7 +51,7 @@ class ServerFileMl { } } -class ServerFaceEmbeddings { +class ServerFaceEmbedding { public faces: ServerFace[]; public version: number; public client: string; @@ -138,10 +138,10 @@ function LocalFileMlDataToServerFileMl( ); faces.push(newFaceObject); } - const faceEmbeddings = new ServerFaceEmbeddings(faces, userAgent, 1); + const faceEmbedding = new ServerFaceEmbedding(faces, userAgent, 1); return new ServerFileMl( localFileMlData.fileID, - faceEmbeddings, + faceEmbedding, localFileMlData.height, localFileMlData.width, ); From 34166ecffb16831793e4446f4e0b3d2b43e57109 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 14:00:00 +0530 Subject: [PATCH 50/77] Move to generator --- web/apps/photos/src/services/face/f-index.ts | 13 +++++++++++-- web/apps/photos/src/services/face/indexer.worker.ts | 4 ++-- web/apps/photos/src/services/face/remote.ts | 10 ++++++---- web/apps/photos/src/services/face/types-old.ts | 1 + 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/web/apps/photos/src/services/face/f-index.ts b/web/apps/photos/src/services/face/f-index.ts index ac7afa5d7d..737a080407 100644 --- a/web/apps/photos/src/services/face/f-index.ts +++ b/web/apps/photos/src/services/face/f-index.ts @@ -46,10 +46,14 @@ import type { Face, FaceDetection, MlFileData } from "./types-old"; * @param file The contents of {@link enteFile} as a web {@link File}, if * available. These are used when they are provided, otherwise the file is * downloaded and decrypted from remote. + * + * @param userAgent The UA of the current client (the client that is generating + * the embedding). */ export const indexFaces = async ( enteFile: EnteFile, file: File | undefined, + userAgent: string, ) => { const startTime = Date.now(); @@ -58,7 +62,7 @@ export const indexFaces = async ( ); let mlFile: MlFileData; try { - mlFile = await indexFaces_(enteFile, imageBitmap); + mlFile = await indexFaces_(enteFile, imageBitmap, userAgent); } finally { imageBitmap.close(); } @@ -98,7 +102,11 @@ const fetchRenderableBlob = async (enteFile: EnteFile) => { } }; -const indexFaces_ = async (enteFile: EnteFile, imageBitmap: ImageBitmap) => { +const indexFaces_ = async ( + enteFile: EnteFile, + imageBitmap: ImageBitmap, + userAgent: string, +) => { const fileID = enteFile.id; const { width, height } = imageBitmap; const imageDimensions = { width, height }; @@ -108,6 +116,7 @@ const indexFaces_ = async (enteFile: EnteFile, imageBitmap: ImageBitmap) => { height, faceEmbedding: { version: 1, + client: userAgent, }, mlVersion: defaultMLVersion, errorCount: 0, diff --git a/web/apps/photos/src/services/face/indexer.worker.ts b/web/apps/photos/src/services/face/indexer.worker.ts index b4b9bc08c7..409dc62483 100644 --- a/web/apps/photos/src/services/face/indexer.worker.ts +++ b/web/apps/photos/src/services/face/indexer.worker.ts @@ -33,7 +33,7 @@ export class FaceIndexerWorker { let faceIndex: MlFileData; try { - faceIndex = await indexFaces(enteFile, file); + faceIndex = await indexFaces(enteFile, file, userAgent); log.debug(() => ({ f, faceIndex })); } catch (e) { // Mark indexing as having failed only if the indexing itself @@ -44,7 +44,7 @@ export class FaceIndexerWorker { throw e; } - await putFaceEmbedding(enteFile, faceIndex, userAgent); + await putFaceEmbedding(enteFile, faceIndex); return faceIndex; } diff --git a/web/apps/photos/src/services/face/remote.ts b/web/apps/photos/src/services/face/remote.ts index 791534cac0..d49a8b13e9 100644 --- a/web/apps/photos/src/services/face/remote.ts +++ b/web/apps/photos/src/services/face/remote.ts @@ -8,9 +8,8 @@ import type { Face, FaceDetection, MlFileData } from "./types-old"; export const putFaceEmbedding = async ( enteFile: EnteFile, mlFileData: MlFileData, - userAgent: string, ) => { - const serverMl = LocalFileMlDataToServerFileMl(mlFileData, userAgent); + const serverMl = LocalFileMlDataToServerFileMl(mlFileData); log.debug(() => ({ t: "Local ML file data", mlFileData })); log.debug(() => ({ t: "Uploaded ML file data", @@ -111,7 +110,6 @@ class ServerFaceBox { function LocalFileMlDataToServerFileMl( localFileMlData: MlFileData, - userAgent: string, ): ServerFileMl { if (localFileMlData.errorCount > 0) { return null; @@ -138,7 +136,11 @@ function LocalFileMlDataToServerFileMl( ); faces.push(newFaceObject); } - const faceEmbedding = new ServerFaceEmbedding(faces, userAgent, 1); + const faceEmbedding = new ServerFaceEmbedding( + faces, + localFileMlData.faceEmbedding.client, + localFileMlData.faceEmbedding.version, + ); return new ServerFileMl( localFileMlData.fileID, faceEmbedding, diff --git a/web/apps/photos/src/services/face/types-old.ts b/web/apps/photos/src/services/face/types-old.ts index d81ebda5b5..11109f19a0 100644 --- a/web/apps/photos/src/services/face/types-old.ts +++ b/web/apps/photos/src/services/face/types-old.ts @@ -23,6 +23,7 @@ export interface MlFileData { height: number; faceEmbedding: { version: number; + client: string; }; mlVersion: number; errorCount: number; From 13d15ceeb9b3a86cf63cfde1a64423464a85a036 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 14:09:59 +0530 Subject: [PATCH 51/77] nest --- web/apps/photos/src/services/face/db-old.ts | 2 +- web/apps/photos/src/services/face/f-index.ts | 21 ++++++++++++------- web/apps/photos/src/services/face/indexer.ts | 14 ++++++------- web/apps/photos/src/services/face/remote.ts | 4 ++-- .../photos/src/services/face/types-old.ts | 2 +- 5 files changed, 25 insertions(+), 18 deletions(-) diff --git a/web/apps/photos/src/services/face/db-old.ts b/web/apps/photos/src/services/face/db-old.ts index 7aabe57ac2..9528e6a8da 100644 --- a/web/apps/photos/src/services/face/db-old.ts +++ b/web/apps/photos/src/services/face/db-old.ts @@ -30,7 +30,7 @@ export interface MinimalPersistedFileData { fileID: number; mlVersion: number; errorCount: number; - faces?: { personId?: number; id: string }[]; + faceEmbedding: { faces?: { id: string }[] }; } interface Config {} diff --git a/web/apps/photos/src/services/face/f-index.ts b/web/apps/photos/src/services/face/f-index.ts index 737a080407..335bdfaf48 100644 --- a/web/apps/photos/src/services/face/f-index.ts +++ b/web/apps/photos/src/services/face/f-index.ts @@ -68,7 +68,7 @@ export const indexFaces = async ( } log.debug(() => { - const nf = mlFile.faces?.length ?? 0; + const nf = mlFile.faceEmbedding.faces?.length ?? 0; const ms = Date.now() - startTime; return `Indexed ${nf} faces in file ${logIdentifier(enteFile)} (${ms} ms)`; }); @@ -128,12 +128,12 @@ const indexFaces_ = async ( fileId: fileID, detection, })); - mlFile.faces = detectedFaces; + mlFile.faceEmbedding.faces = detectedFaces; if (detectedFaces.length > 0) { const alignments: FaceAlignment[] = []; - for (const face of mlFile.faces) { + for (const face of mlFile.faceEmbedding.faces) { const alignment = computeFaceAlignment(face.detection); alignments.push(alignment); @@ -147,13 +147,20 @@ const indexFaces_ = async ( alignments, ); - const blurValues = detectBlur(alignedFacesData, mlFile.faces); - mlFile.faces.forEach((f, i) => (f.blurValue = blurValues[i])); + const blurValues = detectBlur( + alignedFacesData, + mlFile.faceEmbedding.faces, + ); + mlFile.faceEmbedding.faces.forEach( + (f, i) => (f.blurValue = blurValues[i]), + ); const embeddings = await computeEmbeddings(alignedFacesData); - mlFile.faces.forEach((f, i) => (f.embedding = embeddings[i])); + mlFile.faceEmbedding.faces.forEach( + (f, i) => (f.embedding = embeddings[i]), + ); - mlFile.faces.forEach((face) => { + mlFile.faceEmbedding.faces.forEach((face) => { face.detection = relativeDetection(face.detection, imageDimensions); }); } diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index f7f3a7ba43..62a4e6d040 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -11,9 +11,8 @@ import machineLearningService, { import mlWorkManager from "services/machineLearning/mlWorkManager"; import type { EnteFile } from "types/file"; import { isInternalUserForML } from "utils/user"; -import { indexedAndIndexableCounts, markIndexingFailed } from "./db"; +import { indexedAndIndexableCounts } from "./db"; import type { IndexStatus, MinimalPersistedFileData } from "./db-old"; -import { indexFaces } from "./f-index"; import { FaceIndexerWorker } from "./indexer.worker"; /** @@ -96,20 +95,21 @@ class FaceIndexer { }, 30 * 1000); return; } - + /* const fileID = item.enteFile.id; try { - const faceIndex = await indexFaces(item.enteFile, item.file); + const faceIndex = await indexFaces(item.enteFile, item.file, userAgent); log.info(`faces in file ${fileID}`, faceIndex); } catch (e) { log.error(`Failed to index faces in file ${fileID}`, e); markIndexingFailed(item.enteFile.id); } - +*/ // Let the runloop drain. await wait(0); // Run again. - this.tick(); + // TODO + // this.tick(); } /** @@ -214,7 +214,7 @@ export const unidentifiedFaceIDs = async ( enteFile: EnteFile, ): Promise<{ id: string }[]> => { const mlFileData = await mlIDbStorage.getFile(enteFile.id); - return mlFileData?.faces ?? []; + return mlFileData?.faceEmbedding.faces ?? []; }; /** diff --git a/web/apps/photos/src/services/face/remote.ts b/web/apps/photos/src/services/face/remote.ts index d49a8b13e9..093ad788ab 100644 --- a/web/apps/photos/src/services/face/remote.ts +++ b/web/apps/photos/src/services/face/remote.ts @@ -116,8 +116,8 @@ function LocalFileMlDataToServerFileMl( } const faces: ServerFace[] = []; - for (let i = 0; i < localFileMlData.faces.length; i++) { - const face: Face = localFileMlData.faces[i]; + for (let i = 0; i < localFileMlData.faceEmbedding.faces.length; i++) { + const face: Face = localFileMlData.faceEmbedding.faces[i]; const faceID = face.id; const embedding = face.embedding; const score = face.detection.probability; diff --git a/web/apps/photos/src/services/face/types-old.ts b/web/apps/photos/src/services/face/types-old.ts index 11109f19a0..f9bec47b84 100644 --- a/web/apps/photos/src/services/face/types-old.ts +++ b/web/apps/photos/src/services/face/types-old.ts @@ -18,12 +18,12 @@ export interface Face { export interface MlFileData { fileID: number; - faces?: Face[]; width: number; height: number; faceEmbedding: { version: number; client: string; + faces?: Face[]; }; mlVersion: number; errorCount: number; From 2abcb709d93e9137650508151fc263249b35a2ac Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 14:11:23 +0530 Subject: [PATCH 52/77] Unused --- web/apps/photos/src/services/face/types-old.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/web/apps/photos/src/services/face/types-old.ts b/web/apps/photos/src/services/face/types-old.ts index f9bec47b84..065b65e20b 100644 --- a/web/apps/photos/src/services/face/types-old.ts +++ b/web/apps/photos/src/services/face/types-old.ts @@ -8,7 +8,6 @@ export interface FaceDetection { } export interface Face { - fileId: number; detection: FaceDetection; id: string; blurValue?: number; From b5c52a4ae2da050149cbdf4fa5945503a2c28fe0 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 14:12:50 +0530 Subject: [PATCH 53/77] id --- web/apps/photos/src/components/ml/PeopleList.tsx | 14 +++++++------- web/apps/photos/src/services/face/crop.ts | 2 +- web/apps/photos/src/services/face/db-old.ts | 2 +- web/apps/photos/src/services/face/indexer.ts | 4 ++-- web/apps/photos/src/services/face/remote.ts | 2 +- web/apps/photos/src/services/face/types-old.ts | 2 +- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/web/apps/photos/src/components/ml/PeopleList.tsx b/web/apps/photos/src/components/ml/PeopleList.tsx index d44d100612..38e70c3a8d 100644 --- a/web/apps/photos/src/components/ml/PeopleList.tsx +++ b/web/apps/photos/src/components/ml/PeopleList.tsx @@ -73,14 +73,14 @@ export function PhotoPeopleList() { } export function UnidentifiedFaces({ file }: { file: EnteFile }) { - const [faces, setFaces] = useState<{ id: string }[]>([]); + const [faceIDs, setFaceIDs] = useState([]); useEffect(() => { let didCancel = false; (async () => { - const faces = await unidentifiedFaceIDs(file); - !didCancel && setFaces(faces); + const faceIDs = await unidentifiedFaceIDs(file); + !didCancel && setFaceIDs(faceIDs); })(); return () => { @@ -88,7 +88,7 @@ export function UnidentifiedFaces({ file }: { file: EnteFile }) { }; }, [file]); - if (faces.length == 0) return <>; + if (faceIDs.length == 0) return <>; return ( <> @@ -96,9 +96,9 @@ export function UnidentifiedFaces({ file }: { file: EnteFile }) { {t("UNIDENTIFIED_FACES")} - {faces.map((face) => ( - - + {faceIDs.map((faceID) => ( + + ))} diff --git a/web/apps/photos/src/services/face/crop.ts b/web/apps/photos/src/services/face/crop.ts index 8a09c1b8e6..6adb58d762 100644 --- a/web/apps/photos/src/services/face/crop.ts +++ b/web/apps/photos/src/services/face/crop.ts @@ -13,7 +13,7 @@ export const saveFaceCrop = async ( faceCrop.close(); const cache = await blobCache("face-crops"); - await cache.put(face.id, blob); + await cache.put(face.faceID, blob); return blob; }; diff --git a/web/apps/photos/src/services/face/db-old.ts b/web/apps/photos/src/services/face/db-old.ts index 9528e6a8da..070b1ce765 100644 --- a/web/apps/photos/src/services/face/db-old.ts +++ b/web/apps/photos/src/services/face/db-old.ts @@ -30,7 +30,7 @@ export interface MinimalPersistedFileData { fileID: number; mlVersion: number; errorCount: number; - faceEmbedding: { faces?: { id: string }[] }; + faceEmbedding: { faces: { faceID: string }[] }; } interface Config {} diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index 62a4e6d040..75212ede71 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -212,9 +212,9 @@ const convertToNewInterface = (indexStatus: IndexStatus) => { */ export const unidentifiedFaceIDs = async ( enteFile: EnteFile, -): Promise<{ id: string }[]> => { +): Promise => { const mlFileData = await mlIDbStorage.getFile(enteFile.id); - return mlFileData?.faceEmbedding.faces ?? []; + return mlFileData?.faceEmbedding.faces.map((f) => f.faceID) ?? []; }; /** diff --git a/web/apps/photos/src/services/face/remote.ts b/web/apps/photos/src/services/face/remote.ts index 093ad788ab..4ecb893999 100644 --- a/web/apps/photos/src/services/face/remote.ts +++ b/web/apps/photos/src/services/face/remote.ts @@ -118,7 +118,7 @@ function LocalFileMlDataToServerFileMl( const faces: ServerFace[] = []; for (let i = 0; i < localFileMlData.faceEmbedding.faces.length; i++) { const face: Face = localFileMlData.faceEmbedding.faces[i]; - const faceID = face.id; + const faceID = face.faceID; const embedding = face.embedding; const score = face.detection.probability; const blur = face.blurValue; diff --git a/web/apps/photos/src/services/face/types-old.ts b/web/apps/photos/src/services/face/types-old.ts index 065b65e20b..b40f0844ec 100644 --- a/web/apps/photos/src/services/face/types-old.ts +++ b/web/apps/photos/src/services/face/types-old.ts @@ -8,8 +8,8 @@ export interface FaceDetection { } export interface Face { + faceID: string; detection: FaceDetection; - id: string; blurValue?: number; embedding?: Float32Array; From 57404e1f4922964506061681bed577b332fab75c Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 14:17:31 +0530 Subject: [PATCH 54/77] id2 --- web/apps/photos/src/services/face/f-index.ts | 4 ++-- web/apps/photos/src/services/face/indexer.ts | 1 + web/apps/photos/src/services/face/types-old.ts | 2 +- .../src/services/machineLearning/machineLearningService.ts | 1 + 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/web/apps/photos/src/services/face/f-index.ts b/web/apps/photos/src/services/face/f-index.ts index 335bdfaf48..a8ab040e49 100644 --- a/web/apps/photos/src/services/face/f-index.ts +++ b/web/apps/photos/src/services/face/f-index.ts @@ -117,6 +117,7 @@ const indexFaces_ = async ( faceEmbedding: { version: 1, client: userAgent, + faces: [], }, mlVersion: defaultMLVersion, errorCount: 0, @@ -124,8 +125,7 @@ const indexFaces_ = async ( const faceDetections = await detectFaces(imageBitmap); const detectedFaces = faceDetections.map((detection) => ({ - id: makeFaceID(fileID, detection, imageDimensions), - fileId: fileID, + faceID: makeFaceID(fileID, detection, imageDimensions), detection, })); mlFile.faceEmbedding.faces = detectedFaces; diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index 75212ede71..7ea723f135 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -272,6 +272,7 @@ export const syncLocalFiles = async (userID: number) => { fileID, mlVersion: 0, errorCount: 0, + faceEmbedding: { faces: [] }, }) as MinimalPersistedFileData, ); await mlIDbStorage.putAllFiles(newFiles, tx); diff --git a/web/apps/photos/src/services/face/types-old.ts b/web/apps/photos/src/services/face/types-old.ts index b40f0844ec..354519f204 100644 --- a/web/apps/photos/src/services/face/types-old.ts +++ b/web/apps/photos/src/services/face/types-old.ts @@ -22,7 +22,7 @@ export interface MlFileData { faceEmbedding: { version: number; client: string; - faces?: Face[]; + faces: Face[]; }; mlVersion: number; errorCount: number; diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index 654358712e..fbff8fd5a4 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -85,6 +85,7 @@ class MachineLearningService { fileID, mlVersion: 0, errorCount: 0, + faceEmbedding: { faces: [] }, } as MinimalPersistedFileData; } From 6b0501e27228dab2540bfbce911ed6711b41b2fc Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 14:30:19 +0530 Subject: [PATCH 55/77] Move the score out --- web/apps/photos/src/services/face/f-index.ts | 71 ++++++++++--------- web/apps/photos/src/services/face/remote.ts | 2 +- .../photos/src/services/face/types-old.ts | 2 +- 3 files changed, 39 insertions(+), 36 deletions(-) diff --git a/web/apps/photos/src/services/face/f-index.ts b/web/apps/photos/src/services/face/f-index.ts index a8ab040e49..5f1039c2f9 100644 --- a/web/apps/photos/src/services/face/f-index.ts +++ b/web/apps/photos/src/services/face/f-index.ts @@ -22,7 +22,7 @@ import { pixelRGBBilinear, warpAffineFloat32List, } from "./image"; -import type { Box, Dimensions } from "./types"; +import type { Box, Dimensions, Point } from "./types"; import type { Face, FaceDetection, MlFileData } from "./types-old"; /** @@ -109,7 +109,7 @@ const indexFaces_ = async ( ) => { const fileID = enteFile.id; const { width, height } = imageBitmap; - const imageDimensions = { width, height }; + const imageBox = { width, height }; const mlFile: MlFileData = { fileID: fileID, width, @@ -123,11 +123,14 @@ const indexFaces_ = async ( errorCount: 0, }; - const faceDetections = await detectFaces(imageBitmap); - const detectedFaces = faceDetections.map((detection) => ({ - faceID: makeFaceID(fileID, detection, imageDimensions), - detection, - })); + const yoloFaceDetections = await detectFaces(imageBitmap); + const detectedFaces = yoloFaceDetections.map( + ({ box, landmarks, score }) => ({ + faceID: makeFaceID(fileID, box, imageBox), + detection: { box, landmarks }, + score, + }), + ); mlFile.faceEmbedding.faces = detectedFaces; if (detectedFaces.length > 0) { @@ -161,7 +164,7 @@ const indexFaces_ = async ( ); mlFile.faceEmbedding.faces.forEach((face) => { - face.detection = relativeDetection(face.detection, imageDimensions); + face.detection = relativeDetection(face.detection, imageBox); }); } @@ -175,14 +178,14 @@ const indexFaces_ = async ( */ const detectFaces = async ( imageBitmap: ImageBitmap, -): Promise => { +): Promise => { const rect = ({ width, height }) => ({ x: 0, y: 0, width, height }); const { yoloInput, yoloSize } = convertToYOLOInputFloat32ChannelsFirst(imageBitmap); const yoloOutput = await workerBridge.detectFaces(yoloInput); const faces = filterExtractDetectionsFromYOLOOutput(yoloOutput); - const faceDetections = transformFaceDetections( + const faceDetections = transformYOLOFaceDetections( faces, rect(yoloSize), rect(imageBitmap), @@ -243,6 +246,12 @@ const convertToYOLOInputFloat32ChannelsFirst = (imageBitmap: ImageBitmap) => { return { yoloInput, yoloSize }; }; +export interface YOLOFaceDetection { + box: Box; + landmarks: Point[]; + score: number; +} + /** * Extract detected faces from the YOLOv5Face's output. * @@ -261,8 +270,8 @@ const convertToYOLOInputFloat32ChannelsFirst = (imageBitmap: ImageBitmap) => { */ const filterExtractDetectionsFromYOLOOutput = ( rows: Float32Array, -): FaceDetection[] => { - const faces: FaceDetection[] = []; +): YOLOFaceDetection[] => { + const faces: YOLOFaceDetection[] = []; // Iterate over each row. for (let i = 0; i < rows.length; i += 16) { const score = rows[i + 4]; @@ -287,7 +296,6 @@ const filterExtractDetectionsFromYOLOOutput = ( const rightMouthY = rows[i + 14]; const box = { x, y, width, height }; - const probability = score as number; const landmarks = [ { x: leftEyeX, y: leftEyeY }, { x: rightEyeX, y: rightEyeY }, @@ -295,26 +303,26 @@ const filterExtractDetectionsFromYOLOOutput = ( { x: leftMouthX, y: leftMouthY }, { x: rightMouthX, y: rightMouthY }, ]; - faces.push({ box, landmarks, probability }); + faces.push({ box, landmarks, score }); } return faces; }; /** - * Transform the given {@link faceDetections} from their coordinate system in + * Transform the given {@link yoloFaceDetections} from their coordinate system in * which they were detected ({@link inBox}) back to the coordinate system of the * original image ({@link toBox}). */ -const transformFaceDetections = ( - faceDetections: FaceDetection[], +const transformYOLOFaceDetections = ( + yoloFaceDetections: YOLOFaceDetection[], inBox: Box, toBox: Box, -): FaceDetection[] => { +): YOLOFaceDetection[] => { const transform = boxTransformationMatrix(inBox, toBox); - return faceDetections.map((f) => ({ + return yoloFaceDetections.map((f) => ({ box: transformBox(f.box, transform), landmarks: f.landmarks.map((p) => applyToPoint(transform, p)), - probability: f.probability, + score: f.score, })); }; @@ -346,8 +354,8 @@ const transformBox = (box: Box, transform: TransformationMatrix): Box => { * Remove overlapping faces from an array of face detections through non-maximum * suppression algorithm. * - * This function sorts the detections by their probability in descending order, - * then iterates over them. + * This function sorts the detections by their score in descending order, then + * iterates over them. * * For each detection, it calculates the Intersection over Union (IoU) with all * other detections. @@ -356,8 +364,8 @@ const transformBox = (box: Box, transform: TransformationMatrix): Box => { * (`iouThreshold`), the other detection is considered overlapping and is * removed. * - * @param detections - An array of face detections to remove overlapping faces - * from. + * @param detections - An array of YOLO face detections to remove overlapping + * faces from. * * @param iouThreshold - The minimum IoU between two detections for them to be * considered overlapping. @@ -365,11 +373,11 @@ const transformBox = (box: Box, transform: TransformationMatrix): Box => { * @returns An array of face detections with overlapping faces removed */ const naiveNonMaxSuppression = ( - detections: FaceDetection[], + detections: YOLOFaceDetection[], iouThreshold: number, -): FaceDetection[] => { +): YOLOFaceDetection[] => { // Sort the detections by score, the highest first. - detections.sort((a, b) => b.probability - a.probability); + detections.sort((a, b) => b.score - a.score); // Loop through the detections and calculate the IOU. for (let i = 0; i < detections.length - 1; i++) { @@ -413,11 +421,7 @@ const intersectionOverUnion = (a: FaceDetection, b: FaceDetection): number => { return intersectionArea / unionArea; }; -const makeFaceID = ( - fileID: number, - { box }: FaceDetection, - image: Dimensions, -) => { +const makeFaceID = (fileID: number, box: Box, image: Dimensions) => { const part = (v: number) => clamp(v, 0.0, 0.999999).toFixed(5).substring(2); const xMin = part(box.x / image.width); const yMin = part(box.y / image.height); @@ -760,6 +764,5 @@ const relativeDetection = ( x: l.x / width, y: l.y / height, })); - const probability = faceDetection.probability; - return { box, landmarks, probability }; + return { box, landmarks }; }; diff --git a/web/apps/photos/src/services/face/remote.ts b/web/apps/photos/src/services/face/remote.ts index 4ecb893999..904eb8a50e 100644 --- a/web/apps/photos/src/services/face/remote.ts +++ b/web/apps/photos/src/services/face/remote.ts @@ -120,7 +120,7 @@ function LocalFileMlDataToServerFileMl( const face: Face = localFileMlData.faceEmbedding.faces[i]; const faceID = face.faceID; const embedding = face.embedding; - const score = face.detection.probability; + const score = face.score; const blur = face.blurValue; const detection: FaceDetection = face.detection; const box = detection.box; diff --git a/web/apps/photos/src/services/face/types-old.ts b/web/apps/photos/src/services/face/types-old.ts index 354519f204..079e7285f4 100644 --- a/web/apps/photos/src/services/face/types-old.ts +++ b/web/apps/photos/src/services/face/types-old.ts @@ -4,12 +4,12 @@ export interface FaceDetection { // box and landmarks is relative to image dimentions stored at mlFileData box: Box; landmarks?: Point[]; - probability?: number; } export interface Face { faceID: string; detection: FaceDetection; + score: number; blurValue?: number; embedding?: Float32Array; From b9a07e536ca5eb8898dd65aa389f44dd52e96dd4 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 14:32:52 +0530 Subject: [PATCH 56/77] blur --- web/apps/photos/src/services/face/f-index.ts | 12 ++++-------- web/apps/photos/src/services/face/remote.ts | 2 +- web/apps/photos/src/services/face/types-old.ts | 2 +- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/web/apps/photos/src/services/face/f-index.ts b/web/apps/photos/src/services/face/f-index.ts index 5f1039c2f9..f219cc7f4e 100644 --- a/web/apps/photos/src/services/face/f-index.ts +++ b/web/apps/photos/src/services/face/f-index.ts @@ -111,7 +111,7 @@ const indexFaces_ = async ( const { width, height } = imageBitmap; const imageBox = { width, height }; const mlFile: MlFileData = { - fileID: fileID, + fileID, width, height, faceEmbedding: { @@ -129,6 +129,7 @@ const indexFaces_ = async ( faceID: makeFaceID(fileID, box, imageBox), detection: { box, landmarks }, score, + blur: 0, }), ); mlFile.faceEmbedding.faces = detectedFaces; @@ -150,13 +151,8 @@ const indexFaces_ = async ( alignments, ); - const blurValues = detectBlur( - alignedFacesData, - mlFile.faceEmbedding.faces, - ); - mlFile.faceEmbedding.faces.forEach( - (f, i) => (f.blurValue = blurValues[i]), - ); + const blurs = detectBlur(alignedFacesData, mlFile.faceEmbedding.faces); + mlFile.faceEmbedding.faces.forEach((f, i) => (f.blur = blurs[i])); const embeddings = await computeEmbeddings(alignedFacesData); mlFile.faceEmbedding.faces.forEach( diff --git a/web/apps/photos/src/services/face/remote.ts b/web/apps/photos/src/services/face/remote.ts index 904eb8a50e..c6b3c84491 100644 --- a/web/apps/photos/src/services/face/remote.ts +++ b/web/apps/photos/src/services/face/remote.ts @@ -121,7 +121,7 @@ function LocalFileMlDataToServerFileMl( const faceID = face.faceID; const embedding = face.embedding; const score = face.score; - const blur = face.blurValue; + const blur = face.blur; const detection: FaceDetection = face.detection; const box = detection.box; const landmarks = detection.landmarks; diff --git a/web/apps/photos/src/services/face/types-old.ts b/web/apps/photos/src/services/face/types-old.ts index 079e7285f4..086fd6dda8 100644 --- a/web/apps/photos/src/services/face/types-old.ts +++ b/web/apps/photos/src/services/face/types-old.ts @@ -10,7 +10,7 @@ export interface Face { faceID: string; detection: FaceDetection; score: number; - blurValue?: number; + blur: number; embedding?: Float32Array; } From 1a292aae2784e15ab9ff74b8e7cb70afaf9d5fe4 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 15:16:44 +0530 Subject: [PATCH 57/77] Split --- web/apps/photos/src/services/face/crop.ts | 5 +- web/apps/photos/src/services/face/f-index.ts | 108 +++++++++++-------- 2 files changed, 68 insertions(+), 45 deletions(-) diff --git a/web/apps/photos/src/services/face/crop.ts b/web/apps/photos/src/services/face/crop.ts index 6adb58d762..ff1d6026af 100644 --- a/web/apps/photos/src/services/face/crop.ts +++ b/web/apps/photos/src/services/face/crop.ts @@ -1,11 +1,10 @@ import { blobCache } from "@/next/blob-cache"; import type { FaceAlignment } from "./f-index"; import type { Box } from "./types"; -import type { Face } from "./types-old"; export const saveFaceCrop = async ( imageBitmap: ImageBitmap, - face: Face, + faceID: string, alignment: FaceAlignment, ) => { const faceCrop = extractFaceCrop(imageBitmap, alignment); @@ -13,7 +12,7 @@ export const saveFaceCrop = async ( faceCrop.close(); const cache = await blobCache("face-crops"); - await cache.put(face.faceID, blob); + await cache.put(faceID, blob); return blob; }; diff --git a/web/apps/photos/src/services/face/f-index.ts b/web/apps/photos/src/services/face/f-index.ts index f219cc7f4e..450e72841c 100644 --- a/web/apps/photos/src/services/face/f-index.ts +++ b/web/apps/photos/src/services/face/f-index.ts @@ -23,7 +23,7 @@ import { warpAffineFloat32List, } from "./image"; import type { Box, Dimensions, Point } from "./types"; -import type { Face, FaceDetection, MlFileData } from "./types-old"; +import type { MlFileData } from "./types-old"; /** * Index faces in the given file. @@ -107,64 +107,81 @@ const indexFaces_ = async ( imageBitmap: ImageBitmap, userAgent: string, ) => { - const fileID = enteFile.id; const { width, height } = imageBitmap; - const imageBox = { width, height }; - const mlFile: MlFileData = { + const fileID = enteFile.id; + return { fileID, width, height, faceEmbedding: { version: 1, client: userAgent, - faces: [], + faces: await indexFacesInBitmap(fileID, imageBitmap), }, mlVersion: defaultMLVersion, errorCount: 0, }; +}; + +const indexFacesInBitmap = async ( + fileID: number, + imageBitmap: ImageBitmap, +): Promise => { + const { width, height } = imageBitmap; + const imageBox = { width, height }; const yoloFaceDetections = await detectFaces(imageBitmap); - const detectedFaces = yoloFaceDetections.map( - ({ box, landmarks, score }) => ({ - faceID: makeFaceID(fileID, box, imageBox), - detection: { box, landmarks }, - score, - blur: 0, - }), + const partialResult = yoloFaceDetections.map( + ({ box, landmarks, score }) => { + const faceID = makeFaceID(fileID, box, imageBox); + const detection = { box, landmarks }; + return { faceID, detection, score }; + }, ); - mlFile.faceEmbedding.faces = detectedFaces; - if (detectedFaces.length > 0) { - const alignments: FaceAlignment[] = []; + const alignments: FaceAlignment[] = []; - for (const face of mlFile.faceEmbedding.faces) { - const alignment = computeFaceAlignment(face.detection); - alignments.push(alignment); + for (const { faceID, detection } of partialResult) { + const alignment = computeFaceAlignment(detection); + alignments.push(alignment); - // This step is not really part of the indexing pipeline, we just do - // it here since we have already computed the face alignment. - await saveFaceCrop(imageBitmap, face, alignment); + // This step is not really part of the indexing pipeline, we just do + // it here since we have already computed the face alignment. Ignore + // errors that happen during this though. + try { + await saveFaceCrop(imageBitmap, faceID, alignment); + } catch (e) { + log.error(`Failed to save face crop for faceID ${faceID}`, e); } + } - const alignedFacesData = convertToMobileFaceNetInput( - imageBitmap, - alignments, - ); + const alignedFacesData = convertToMobileFaceNetInput( + imageBitmap, + alignments, + ); - const blurs = detectBlur(alignedFacesData, mlFile.faceEmbedding.faces); - mlFile.faceEmbedding.faces.forEach((f, i) => (f.blur = blurs[i])); + const embeddings = await computeEmbeddings(alignedFacesData); + const blurs = detectBlur( + alignedFacesData, + partialResult.map((f) => f.detection), + ); - const embeddings = await computeEmbeddings(alignedFacesData); - mlFile.faceEmbedding.faces.forEach( - (f, i) => (f.embedding = embeddings[i]), - ); + const faces = []; - mlFile.faceEmbedding.faces.forEach((face) => { - face.detection = relativeDetection(face.detection, imageBox); + for (let i = 0; i < partialResult.length; i++) { + const { faceID, detection, score } = partialResult[i]; + const blur = blurs[i]; + const embedding = embeddings[i]; + faces.push({ + faceID, + detection: relativeDetection(detection, imageBox), + score, + blur, + embedding, }); } - return mlFile; + return faces; }; /** @@ -534,28 +551,35 @@ const convertToMobileFaceNetInput = ( return faceData; }; +interface FaceDetection { + box: Box; + landmarks: Point[]; +} + /** * Laplacian blur detection. * - * Return an array of detected blur values, one for each face in {@link faces}. - * The face data is taken from the slice of {@link alignedFacesData} - * corresponding to each face of {@link faces}. + * Return an array of detected blur values, one for each face detection in + * {@link faceDetections}. The face data is taken from the slice of + * {@link alignedFacesData} corresponding to the face of {@link faceDetections}. */ -const detectBlur = (alignedFacesData: Float32Array, faces: Face[]): number[] => - faces.map((face, i) => { +const detectBlur = ( + alignedFacesData: Float32Array, + faceDetections: FaceDetection[], +): number[] => + faceDetections.map((d, i) => { const faceImage = grayscaleIntMatrixFromNormalized2List( alignedFacesData, i, mobileFaceNetFaceSize, mobileFaceNetFaceSize, ); - return matrixVariance(applyLaplacian(faceImage, faceDirection(face))); + return matrixVariance(applyLaplacian(faceImage, faceDirection(d))); }); type FaceDirection = "left" | "right" | "straight"; -const faceDirection = (face: Face): FaceDirection => { - const landmarks = face.detection.landmarks; +const faceDirection = ({ landmarks }: FaceDetection): FaceDirection => { const leftEye = landmarks[0]; const rightEye = landmarks[1]; const nose = landmarks[2]; From a6a0a24b26651f91b88aec0bc37d4f2e95666c25 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 15:23:31 +0530 Subject: [PATCH 58/77] Refer --- web/apps/photos/src/services/face/db-old.ts | 5 +++- .../photos/src/services/face/types-old.ts | 29 ++----------------- 2 files changed, 7 insertions(+), 27 deletions(-) diff --git a/web/apps/photos/src/services/face/db-old.ts b/web/apps/photos/src/services/face/db-old.ts index 070b1ce765..b66c72d811 100644 --- a/web/apps/photos/src/services/face/db-old.ts +++ b/web/apps/photos/src/services/face/db-old.ts @@ -119,7 +119,10 @@ class MLIDbStorage { if (oldVersion < 1) { const filesStore = db.createObjectStore("files", { - keyPath: "fileId", + // keyPath: "fileId", TODO(MR): Changing this, since + // we're going to be deleting this DB before this PR is + // merged. + keyPath: "fileID", }); filesStore.createIndex("mlVersion", [ "mlVersion", diff --git a/web/apps/photos/src/services/face/types-old.ts b/web/apps/photos/src/services/face/types-old.ts index 086fd6dda8..d2bf61a4f8 100644 --- a/web/apps/photos/src/services/face/types-old.ts +++ b/web/apps/photos/src/services/face/types-old.ts @@ -1,29 +1,6 @@ -import type { Box, Point } from "./types"; +import type { FaceIndex } from "./types"; -export interface FaceDetection { - // box and landmarks is relative to image dimentions stored at mlFileData - box: Box; - landmarks?: Point[]; -} - -export interface Face { - faceID: string; - detection: FaceDetection; - score: number; - blur: number; - - embedding?: Float32Array; -} - -export interface MlFileData { - fileID: number; - width: number; - height: number; - faceEmbedding: { - version: number; - client: string; - faces: Face[]; - }; +export type MlFileData = FaceIndex & { mlVersion: number; errorCount: number; -} +}; From 2b3b84de0f3dbce303a64fd927854153708dfa93 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 15:31:07 +0530 Subject: [PATCH 59/77] Closer --- web/apps/photos/src/services/face/f-index.ts | 27 +++++++------------- web/apps/photos/src/services/face/remote.ts | 22 ++++++++-------- 2 files changed, 20 insertions(+), 29 deletions(-) diff --git a/web/apps/photos/src/services/face/f-index.ts b/web/apps/photos/src/services/face/f-index.ts index 450e72841c..690f383b22 100644 --- a/web/apps/photos/src/services/face/f-index.ts +++ b/web/apps/photos/src/services/face/f-index.ts @@ -22,7 +22,7 @@ import { pixelRGBBilinear, warpAffineFloat32List, } from "./image"; -import type { Box, Dimensions, Point } from "./types"; +import type { Box, Dimensions, Face, Point } from "./types"; import type { MlFileData } from "./types-old"; /** @@ -126,7 +126,7 @@ const indexFaces_ = async ( const indexFacesInBitmap = async ( fileID: number, imageBitmap: ImageBitmap, -): Promise => { +): Promise => { const { width, height } = imageBitmap; const imageBox = { width, height }; @@ -166,22 +166,13 @@ const indexFacesInBitmap = async ( partialResult.map((f) => f.detection), ); - const faces = []; - - for (let i = 0; i < partialResult.length; i++) { - const { faceID, detection, score } = partialResult[i]; - const blur = blurs[i]; - const embedding = embeddings[i]; - faces.push({ - faceID, - detection: relativeDetection(detection, imageBox), - score, - blur, - embedding, - }); - } - - return faces; + return partialResult.map(({ faceID, detection, score }, i) => ({ + faceID, + detection: relativeDetection(detection, imageBox), + score, + blur: blurs[i], + embedding: Array.from(embeddings[i]), + })); }; /** diff --git a/web/apps/photos/src/services/face/remote.ts b/web/apps/photos/src/services/face/remote.ts index c6b3c84491..b1ce10366d 100644 --- a/web/apps/photos/src/services/face/remote.ts +++ b/web/apps/photos/src/services/face/remote.ts @@ -3,7 +3,7 @@ import ComlinkCryptoWorker from "@ente/shared/crypto"; import { putEmbedding } from "services/embeddingService"; import type { EnteFile } from "types/file"; import type { Point } from "./types"; -import type { Face, FaceDetection, MlFileData } from "./types-old"; +import type { MlFileData } from "./types-old"; export const putFaceEmbedding = async ( enteFile: EnteFile, @@ -117,24 +117,24 @@ function LocalFileMlDataToServerFileMl( const faces: ServerFace[] = []; for (let i = 0; i < localFileMlData.faceEmbedding.faces.length; i++) { - const face: Face = localFileMlData.faceEmbedding.faces[i]; + const face = localFileMlData.faceEmbedding.faces[i]; const faceID = face.faceID; const embedding = face.embedding; const score = face.score; const blur = face.blur; - const detection: FaceDetection = face.detection; + const detection = face.detection; const box = detection.box; const landmarks = detection.landmarks; - const newBox = new ServerFaceBox(box.x, box.y, box.width, box.height); - const newFaceObject = new ServerFace( - faceID, - Array.from(embedding), - new ServerDetection(newBox, landmarks), - score, - blur, + faces.push( + new ServerFace( + faceID, + Array.from(embedding), + new ServerDetection(box, landmarks), + score, + blur, + ), ); - faces.push(newFaceObject); } const faceEmbedding = new ServerFaceEmbedding( faces, From 9dbec2729cb0ddac0e381f80ea8febea4ff0e9a5 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 15:44:25 +0530 Subject: [PATCH 60/77] remote mapping --- web/apps/photos/src/services/face/f-index.ts | 22 +-- .../src/services/face/indexer.worker.ts | 23 +-- web/apps/photos/src/services/face/remote.ts | 137 +----------------- web/apps/photos/src/utils/file/index.ts | 4 +- 4 files changed, 32 insertions(+), 154 deletions(-) diff --git a/web/apps/photos/src/services/face/f-index.ts b/web/apps/photos/src/services/face/f-index.ts index 690f383b22..feb48e6f1f 100644 --- a/web/apps/photos/src/services/face/f-index.ts +++ b/web/apps/photos/src/services/face/f-index.ts @@ -14,7 +14,7 @@ import { translate, } from "transformation-matrix"; import type { EnteFile } from "types/file"; -import { getRenderableImage, logIdentifier } from "utils/file"; +import { fileLogID, getRenderableImage } from "utils/file"; import { saveFaceCrop } from "./crop"; import { clamp, @@ -22,8 +22,7 @@ import { pixelRGBBilinear, warpAffineFloat32List, } from "./image"; -import type { Box, Dimensions, Face, Point } from "./types"; -import type { MlFileData } from "./types-old"; +import type { Box, Dimensions, Face, FaceIndex, Point } from "./types"; /** * Index faces in the given file. @@ -60,19 +59,20 @@ export const indexFaces = async ( const imageBitmap = await renderableImageBlob(enteFile, file).then( createImageBitmap, ); - let mlFile: MlFileData; + + let index: FaceIndex; try { - mlFile = await indexFaces_(enteFile, imageBitmap, userAgent); + index = await indexFaces_(enteFile, imageBitmap, userAgent); } finally { imageBitmap.close(); } log.debug(() => { - const nf = mlFile.faceEmbedding.faces?.length ?? 0; + const nf = index.faceEmbedding.faces.length; const ms = Date.now() - startTime; - return `Indexed ${nf} faces in file ${logIdentifier(enteFile)} (${ms} ms)`; + return `Indexed ${nf} faces in ${fileLogID(enteFile)} (${ms} ms)`; }); - return mlFile; + return index; }; /** @@ -145,9 +145,9 @@ const indexFacesInBitmap = async ( const alignment = computeFaceAlignment(detection); alignments.push(alignment); - // This step is not really part of the indexing pipeline, we just do - // it here since we have already computed the face alignment. Ignore - // errors that happen during this though. + // This step is not part of the indexing pipeline, we just do it here + // since we have already computed the face alignment. Ignore errors that + // happen during this since it does not impact the generated face index. try { await saveFaceCrop(imageBitmap, faceID, alignment); } catch (e) { diff --git a/web/apps/photos/src/services/face/indexer.worker.ts b/web/apps/photos/src/services/face/indexer.worker.ts index 409dc62483..0dccd61780 100644 --- a/web/apps/photos/src/services/face/indexer.worker.ts +++ b/web/apps/photos/src/services/face/indexer.worker.ts @@ -1,10 +1,14 @@ import log from "@/next/log"; -import { putFaceEmbedding } from "services/face/remote"; import type { EnteFile } from "types/file"; -import { logIdentifier } from "utils/file"; -import { closeFaceDBConnectionsIfNeeded, markIndexingFailed } from "./db"; +import { fileLogID } from "utils/file"; +import { + closeFaceDBConnectionsIfNeeded, + markIndexingFailed, + saveFaceIndex, +} from "./db"; import { indexFaces } from "./f-index"; -import type { MlFileData } from "./types-old"; +import { putFaceIndex } from "./remote"; +import type { FaceIndex } from "./types"; /** * Index faces in a file, save the persist the results locally, and put them on @@ -29,22 +33,21 @@ export class FaceIndexerWorker { * downloaded and decrypted from remote. */ async index(enteFile: EnteFile, file: File | undefined, userAgent: string) { - const f = logIdentifier(enteFile); - - let faceIndex: MlFileData; + let faceIndex: FaceIndex; try { faceIndex = await indexFaces(enteFile, file, userAgent); - log.debug(() => ({ f, faceIndex })); } catch (e) { // Mark indexing as having failed only if the indexing itself // failed, not if there were subsequent failures (like when trying // to put the result to remote or save it to the local face DB). - log.error(`Failed to index faces in file ${f}`, e); + log.error(`Failed to index faces in ${fileLogID(enteFile)}`, e); markIndexingFailed(enteFile.id); throw e; } - await putFaceEmbedding(enteFile, faceIndex); + await putFaceIndex(enteFile, faceIndex); + await saveFaceIndex(faceIndex); + return faceIndex; } diff --git a/web/apps/photos/src/services/face/remote.ts b/web/apps/photos/src/services/face/remote.ts index b1ce10366d..2fd50024da 100644 --- a/web/apps/photos/src/services/face/remote.ts +++ b/web/apps/photos/src/services/face/remote.ts @@ -2,23 +2,20 @@ import log from "@/next/log"; import ComlinkCryptoWorker from "@ente/shared/crypto"; import { putEmbedding } from "services/embeddingService"; import type { EnteFile } from "types/file"; -import type { Point } from "./types"; -import type { MlFileData } from "./types-old"; +import type { FaceIndex } from "./types"; -export const putFaceEmbedding = async ( +export const putFaceIndex = async ( enteFile: EnteFile, - mlFileData: MlFileData, + faceIndex: FaceIndex, ) => { - const serverMl = LocalFileMlDataToServerFileMl(mlFileData); - log.debug(() => ({ t: "Local ML file data", mlFileData })); log.debug(() => ({ - t: "Uploaded ML file data", - d: JSON.stringify(serverMl), + t: "Uploading faceEmbedding", + d: JSON.stringify(faceIndex), })); const comlinkCryptoWorker = await ComlinkCryptoWorker.getInstance(); const { file: encryptedEmbeddingData } = - await comlinkCryptoWorker.encryptMetadata(serverMl, enteFile.key); + await comlinkCryptoWorker.encryptMetadata(faceIndex, enteFile.key); await putEmbedding({ fileID: enteFile.id, encryptedEmbedding: encryptedEmbeddingData.encryptedData, @@ -26,125 +23,3 @@ export const putFaceEmbedding = async ( model: "file-ml-clip-face", }); }; - -export interface FileML extends ServerFileMl { - updatedAt: number; -} - -class ServerFileMl { - public fileID: number; - public height?: number; - public width?: number; - public faceEmbedding: ServerFaceEmbedding; - - public constructor( - fileID: number, - faceEmbedding: ServerFaceEmbedding, - height?: number, - width?: number, - ) { - this.fileID = fileID; - this.height = height; - this.width = width; - this.faceEmbedding = faceEmbedding; - } -} - -class ServerFaceEmbedding { - public faces: ServerFace[]; - public version: number; - public client: string; - - public constructor(faces: ServerFace[], client: string, version: number) { - this.faces = faces; - this.client = client; - this.version = version; - } -} - -class ServerFace { - public faceID: string; - public embedding: number[]; - public detection: ServerDetection; - public score: number; - public blur: number; - - public constructor( - faceID: string, - embedding: number[], - detection: ServerDetection, - score: number, - blur: number, - ) { - this.faceID = faceID; - this.embedding = embedding; - this.detection = detection; - this.score = score; - this.blur = blur; - } -} - -class ServerDetection { - public box: ServerFaceBox; - public landmarks: Point[]; - - public constructor(box: ServerFaceBox, landmarks: Point[]) { - this.box = box; - this.landmarks = landmarks; - } -} - -class ServerFaceBox { - public x: number; - public y: number; - public width: number; - public height: number; - - public constructor(x: number, y: number, width: number, height: number) { - this.x = x; - this.y = y; - this.width = width; - this.height = height; - } -} - -function LocalFileMlDataToServerFileMl( - localFileMlData: MlFileData, -): ServerFileMl { - if (localFileMlData.errorCount > 0) { - return null; - } - - const faces: ServerFace[] = []; - for (let i = 0; i < localFileMlData.faceEmbedding.faces.length; i++) { - const face = localFileMlData.faceEmbedding.faces[i]; - const faceID = face.faceID; - const embedding = face.embedding; - const score = face.score; - const blur = face.blur; - const detection = face.detection; - const box = detection.box; - const landmarks = detection.landmarks; - - faces.push( - new ServerFace( - faceID, - Array.from(embedding), - new ServerDetection(box, landmarks), - score, - blur, - ), - ); - } - const faceEmbedding = new ServerFaceEmbedding( - faces, - localFileMlData.faceEmbedding.client, - localFileMlData.faceEmbedding.version, - ); - return new ServerFileMl( - localFileMlData.fileID, - faceEmbedding, - localFileMlData.height, - localFileMlData.width, - ); -} diff --git a/web/apps/photos/src/utils/file/index.ts b/web/apps/photos/src/utils/file/index.ts index c15cca63c0..2879bdd757 100644 --- a/web/apps/photos/src/utils/file/index.ts +++ b/web/apps/photos/src/utils/file/index.ts @@ -86,8 +86,8 @@ const moduleState = new ModuleState(); * given {@link enteFile}. The returned string contains the file name (for ease * of debugging) and the file ID (for exactness). */ -export const logIdentifier = (enteFile: EnteFile) => - `${enteFile.metadata.title ?? "-"} (${enteFile.id})`; +export const fileLogID = (enteFile: EnteFile) => + `file ${enteFile.metadata.title ?? "-"} (${enteFile.id})`; export async function getUpdatedEXIFFileForDownload( fileReader: FileReader, From 6327a7f9da749c6b8979b8a02ee89927d7c38988 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 15:54:01 +0530 Subject: [PATCH 61/77] nu --- web/apps/photos/src/services/face/f-index.ts | 50 ++++++------------- .../src/services/face/indexer.worker.ts | 20 ++++++-- .../machineLearning/machineLearningService.ts | 29 +---------- 3 files changed, 34 insertions(+), 65 deletions(-) diff --git a/web/apps/photos/src/services/face/f-index.ts b/web/apps/photos/src/services/face/f-index.ts index feb48e6f1f..4bf829ff6f 100644 --- a/web/apps/photos/src/services/face/f-index.ts +++ b/web/apps/photos/src/services/face/f-index.ts @@ -14,7 +14,7 @@ import { translate, } from "transformation-matrix"; import type { EnteFile } from "types/file"; -import { fileLogID, getRenderableImage } from "utils/file"; +import { getRenderableImage } from "utils/file"; import { saveFaceCrop } from "./crop"; import { clamp, @@ -22,7 +22,7 @@ import { pixelRGBBilinear, warpAffineFloat32List, } from "./image"; -import type { Box, Dimensions, Face, FaceIndex, Point } from "./types"; +import type { Box, Dimensions, Face, Point } from "./types"; /** * Index faces in the given file. @@ -54,25 +54,28 @@ export const indexFaces = async ( file: File | undefined, userAgent: string, ) => { - const startTime = Date.now(); - const imageBitmap = await renderableImageBlob(enteFile, file).then( createImageBitmap, ); + const { width, height } = imageBitmap; + const fileID = enteFile.id; - let index: FaceIndex; try { - index = await indexFaces_(enteFile, imageBitmap, userAgent); + return { + fileID, + width, + height, + faceEmbedding: { + version: 1, + client: userAgent, + faces: await indexFacesInBitmap(fileID, imageBitmap), + }, + mlVersion: defaultMLVersion, + errorCount: 0, + }; } finally { imageBitmap.close(); } - - log.debug(() => { - const nf = index.faceEmbedding.faces.length; - const ms = Date.now() - startTime; - return `Indexed ${nf} faces in ${fileLogID(enteFile)} (${ms} ms)`; - }); - return index; }; /** @@ -102,27 +105,6 @@ const fetchRenderableBlob = async (enteFile: EnteFile) => { } }; -const indexFaces_ = async ( - enteFile: EnteFile, - imageBitmap: ImageBitmap, - userAgent: string, -) => { - const { width, height } = imageBitmap; - const fileID = enteFile.id; - return { - fileID, - width, - height, - faceEmbedding: { - version: 1, - client: userAgent, - faces: await indexFacesInBitmap(fileID, imageBitmap), - }, - mlVersion: defaultMLVersion, - errorCount: 0, - }; -}; - const indexFacesInBitmap = async ( fileID: number, imageBitmap: ImageBitmap, diff --git a/web/apps/photos/src/services/face/indexer.worker.ts b/web/apps/photos/src/services/face/indexer.worker.ts index 0dccd61780..969a800295 100644 --- a/web/apps/photos/src/services/face/indexer.worker.ts +++ b/web/apps/photos/src/services/face/indexer.worker.ts @@ -33,6 +33,9 @@ export class FaceIndexerWorker { * downloaded and decrypted from remote. */ async index(enteFile: EnteFile, file: File | undefined, userAgent: string) { + const f = fileLogID(enteFile); + const startTime = Date.now(); + let faceIndex: FaceIndex; try { faceIndex = await indexFaces(enteFile, file, userAgent); @@ -40,13 +43,24 @@ export class FaceIndexerWorker { // Mark indexing as having failed only if the indexing itself // failed, not if there were subsequent failures (like when trying // to put the result to remote or save it to the local face DB). - log.error(`Failed to index faces in ${fileLogID(enteFile)}`, e); + log.error(`Failed to index faces in ${f}`, e); markIndexingFailed(enteFile.id); throw e; } - await putFaceIndex(enteFile, faceIndex); - await saveFaceIndex(faceIndex); + try { + await putFaceIndex(enteFile, faceIndex); + await saveFaceIndex(faceIndex); + } catch (e) { + log.error(`Failed to put/save face index for ${f}`, e); + throw e; + } + + log.debug(() => { + const nf = faceIndex.faceEmbedding.faces.length; + const ms = Date.now() - startTime; + return `Indexed ${nf} faces in ${f} (${ms} ms)`; + }); return faceIndex; } diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index fbff8fd5a4..7e740b7d78 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -235,15 +235,7 @@ class MachineLearningService { await this.syncFile(enteFile, localFile, syncContext.userAgent); syncContext.nSyncedFiles += 1; } catch (e) { - log.error("ML syncFile failed", e); let error = e; - console.error( - "Error in ml sync, fileId: ", - enteFile.id, - "name: ", - enteFile.metadata.title, - error, - ); if ("status" in error) { const parsedMessage = parseUploadErrorCodes(error); error = parsedMessage; @@ -258,7 +250,6 @@ class MachineLearningService { throw error; } - await this.persistMLFileSyncError(enteFile, error); syncContext.nSyncedFiles += 1; } } @@ -275,25 +266,7 @@ class MachineLearningService { const worker = new FaceIndexerWorker(); - const newMlFile = await worker.index(enteFile, localFile, userAgent); - await mlIDbStorage.putFile(newMlFile); - } - - private async persistMLFileSyncError(enteFile: EnteFile, e: Error) { - try { - await mlIDbStorage.upsertFileInTx(enteFile.id, (mlFileData) => { - if (!mlFileData) { - mlFileData = this.newMlData(enteFile.id); - } - mlFileData.errorCount = (mlFileData.errorCount || 0) + 1; - console.error(`lastError for ${enteFile.id}`, e); - - return mlFileData; - }); - } catch (e) { - // TODO: logError or stop sync job after most of the requests are failed - console.error("Error while storing ml sync error", e); - } + await worker.index(enteFile, localFile, userAgent); } } From 1f6be04bf4c48558aad45ac99534f15023144331 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 15:57:27 +0530 Subject: [PATCH 62/77] Rename --- web/apps/photos/src/services/face/f-index.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/web/apps/photos/src/services/face/f-index.ts b/web/apps/photos/src/services/face/f-index.ts index 4bf829ff6f..a0faaea34b 100644 --- a/web/apps/photos/src/services/face/f-index.ts +++ b/web/apps/photos/src/services/face/f-index.ts @@ -110,12 +110,12 @@ const indexFacesInBitmap = async ( imageBitmap: ImageBitmap, ): Promise => { const { width, height } = imageBitmap; - const imageBox = { width, height }; + const imageDimensions = { width, height }; const yoloFaceDetections = await detectFaces(imageBitmap); const partialResult = yoloFaceDetections.map( ({ box, landmarks, score }) => { - const faceID = makeFaceID(fileID, box, imageBox); + const faceID = makeFaceID(fileID, box, imageDimensions); const detection = { box, landmarks }; return { faceID, detection, score }; }, @@ -150,7 +150,7 @@ const indexFacesInBitmap = async ( return partialResult.map(({ faceID, detection, score }, i) => ({ faceID, - detection: relativeDetection(detection, imageBox), + detection: normalizeToImageDimensions(detection, imageDimensions), score, blur: blurs[i], embedding: Array.from(embeddings[i]), @@ -742,7 +742,7 @@ const computeEmbeddings = async ( /** * Convert the coordinates to between 0-1, normalized by the image's dimensions. */ -const relativeDetection = ( +const normalizeToImageDimensions = ( faceDetection: FaceDetection, { width, height }: Dimensions, ): FaceDetection => { From 45d7e3da2cfe26bfc89f865a8b9513f8c766a366 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 16:00:09 +0530 Subject: [PATCH 63/77] Prune --- .../machineLearning/machineLearningService.ts | 32 ++----------------- 1 file changed, 2 insertions(+), 30 deletions(-) diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index 7e740b7d78..47517b4e02 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -1,12 +1,9 @@ import log from "@/next/log"; import { CustomError, parseUploadErrorCodes } from "@ente/shared/error"; import PQueue from "p-queue"; -import mlIDbStorage, { - type MinimalPersistedFileData, -} from "services/face/db-old"; +import mlIDbStorage from "services/face/db-old"; import { syncLocalFiles } from "services/face/indexer"; import { FaceIndexerWorker } from "services/face/indexer.worker"; -import { getLocalFiles } from "services/fileService"; import { EnteFile } from "types/file"; export const defaultMLVersion = 1; @@ -80,31 +77,6 @@ class MachineLearningService { return !error && nOutOfSyncFiles > 0; } - private newMlData(fileID: number) { - return { - fileID, - mlVersion: 0, - errorCount: 0, - faceEmbedding: { faces: [] }, - } as MinimalPersistedFileData; - } - - private async getLocalFilesMap(syncContext: MLSyncContext) { - if (!syncContext.localFilesMap) { - const localFiles = await getLocalFiles(); - - const personalFiles = localFiles.filter( - (f) => f.ownerID === syncContext.userID, - ); - syncContext.localFilesMap = new Map(); - personalFiles.forEach((f) => - syncContext.localFilesMap.set(f.id, f), - ); - } - - return syncContext.localFilesMap; - } - private async getOutOfSyncFiles(syncContext: MLSyncContext) { const startTime = Date.now(); const fileIds = await mlIDbStorage.getFileIds( @@ -115,7 +87,7 @@ class MachineLearningService { log.info("fileIds: ", JSON.stringify(fileIds)); - const localFilesMap = await this.getLocalFilesMap(syncContext); + const localFilesMap = syncContext.localFilesMap; syncContext.outOfSyncFiles = fileIds.map((fileId) => localFilesMap.get(fileId), ); From 46761622f1c14c48586269f253e72e7864c7db8f Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 16:05:50 +0530 Subject: [PATCH 64/77] Fix --- web/apps/photos/src/services/embeddingService.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/web/apps/photos/src/services/embeddingService.ts b/web/apps/photos/src/services/embeddingService.ts index 56cebe5a03..fb77609258 100644 --- a/web/apps/photos/src/services/embeddingService.ts +++ b/web/apps/photos/src/services/embeddingService.ts @@ -7,7 +7,6 @@ import HTTPService from "@ente/shared/network/HTTPService"; import { getEndpoint } from "@ente/shared/network/api"; import localForage from "@ente/shared/storage/localForage"; import { getToken } from "@ente/shared/storage/localStorage/helpers"; -import { FileML } from "services/face/remote"; import type { Embedding, EmbeddingModel, @@ -17,9 +16,13 @@ import type { } from "types/embedding"; import { EnteFile } from "types/file"; import { getLocalCollections } from "./collectionService"; +import type { FaceIndex } from "./face/types"; import { getAllLocalFiles } from "./fileService"; import { getLocalTrashedFiles } from "./trashService"; +type FileML = FaceIndex & { + updatedAt: number; +}; const DIFF_LIMIT = 500; /** Local storage key suffix for embedding sync times */ From 074e867886694ce5f20cdeff63e1dc0576fbff74 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 16:08:07 +0530 Subject: [PATCH 65/77] Disable the download for now --- web/apps/photos/src/pages/gallery/index.tsx | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/web/apps/photos/src/pages/gallery/index.tsx b/web/apps/photos/src/pages/gallery/index.tsx index 4a3fbe9b52..d375e48fc8 100644 --- a/web/apps/photos/src/pages/gallery/index.tsx +++ b/web/apps/photos/src/pages/gallery/index.tsx @@ -85,10 +85,7 @@ import { getSectionSummaries, } from "services/collectionService"; import downloadManager from "services/download"; -import { - syncCLIPEmbeddings, - syncFaceEmbeddings, -} from "services/embeddingService"; +import { syncCLIPEmbeddings } from "services/embeddingService"; import { syncEntities } from "services/entityService"; import { getLocalFiles, syncFiles } from "services/fileService"; import locationSearchService from "services/locationSearchService"; @@ -130,7 +127,6 @@ import { } from "utils/file"; import { isArchivedFile } from "utils/magicMetadata"; import { getSessionExpiredMessage } from "utils/ui"; -import { isInternalUserForML } from "utils/user"; import { getLocalFamilyData } from "utils/user/family"; export const DeadCenter = styled("div")` @@ -720,7 +716,9 @@ export default function Gallery() { const electron = globalThis.electron; if (electron) { await syncCLIPEmbeddings(); - if (isInternalUserForML()) await syncFaceEmbeddings(); + // TODO-ML(MR): Disable fetch until we start storing it in the + // same place as the local ones. + // if (isInternalUserForML()) await syncFaceEmbeddings(); } if (clipService.isPlatformSupported()) { void clipService.scheduleImageEmbeddingExtraction(); From 966e5527ec40f8b33fa6a214b2e51c2db8ab7c5d Mon Sep 17 00:00:00 2001 From: Zxhir <98621617+Imzxhir@users.noreply.github.com> Date: Thu, 30 May 2024 12:05:35 +0100 Subject: [PATCH 66/77] Add alt names for some services --- auth/assets/custom-icons/_data/custom-icons.json | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/auth/assets/custom-icons/_data/custom-icons.json b/auth/assets/custom-icons/_data/custom-icons.json index b911183e88..2a3a8f21a7 100644 --- a/auth/assets/custom-icons/_data/custom-icons.json +++ b/auth/assets/custom-icons/_data/custom-icons.json @@ -32,7 +32,10 @@ }, { "title": "Bloom Host", - "slug": "bloom_host" + "slug": "bloom_host", + "altNames": [ + "Bloom Host Billing" + ] }, { "title": "BorgBase", @@ -345,7 +348,10 @@ "hex": "FFFFFF" }, { - "title": "Techlore" + "title": "Techlore", + "altNames": [ + "Techlore Courses" + ] }, { "title": "Termius", From 27523e2f109e17373ca097e33dd987ec5fdfd1df Mon Sep 17 00:00:00 2001 From: Vishnu Mohandas Date: Thu, 30 May 2024 18:18:28 +0530 Subject: [PATCH 67/77] Update intl_en.arb --- mobile/lib/l10n/intl_en.arb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/lib/l10n/intl_en.arb b/mobile/lib/l10n/intl_en.arb index feefd5a298..6a1a5c67c1 100644 --- a/mobile/lib/l10n/intl_en.arb +++ b/mobile/lib/l10n/intl_en.arb @@ -409,7 +409,7 @@ "manageDeviceStorage": "Manage device storage", "machineLearning": "Machine learning", "magicSearch": "Magic search", - "mlIndexingDescription": "Please note that ML indexing will result in a higher bandwidth and battery usage until all items are indexed.", + "mlIndexingDescription": "Please note that machine learning will result in a higher bandwidth and battery usage.", "loadingModel": "Downloading models...", "waitingForWifi": "Waiting for WiFi...", "status": "Status", From 9f2d770bc2b5b89d1b0270b75956dfa1964d88ec Mon Sep 17 00:00:00 2001 From: Laurens Priem <81471280+laurenspriem@users.noreply.github.com> Date: Thu, 30 May 2024 18:24:43 +0530 Subject: [PATCH 68/77] Update intl_en.arb --- mobile/lib/l10n/intl_en.arb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/lib/l10n/intl_en.arb b/mobile/lib/l10n/intl_en.arb index 6a1a5c67c1..ee2877861b 100644 --- a/mobile/lib/l10n/intl_en.arb +++ b/mobile/lib/l10n/intl_en.arb @@ -409,7 +409,7 @@ "manageDeviceStorage": "Manage device storage", "machineLearning": "Machine learning", "magicSearch": "Magic search", - "mlIndexingDescription": "Please note that machine learning will result in a higher bandwidth and battery usage.", + "mlIndexingDescription": "Please note that machine learning will result in a higher bandwidth and battery usage until all items are indexed.", "loadingModel": "Downloading models...", "waitingForWifi": "Waiting for WiFi...", "status": "Status", From ce93ce65293a74ac527fa4ad97996c234be6b244 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 18:32:43 +0530 Subject: [PATCH 69/77] sync + fetch --- web/apps/photos/src/services/face/db.ts | 6 +- web/apps/photos/src/services/face/indexer.ts | 90 +++++-------------- .../machineLearning/machineLearningService.ts | 24 +---- 3 files changed, 28 insertions(+), 92 deletions(-) diff --git a/web/apps/photos/src/services/face/db.ts b/web/apps/photos/src/services/face/db.ts index fe5a93cdb2..21dd3de41d 100644 --- a/web/apps/photos/src/services/face/db.ts +++ b/web/apps/photos/src/services/face/db.ts @@ -264,11 +264,13 @@ export const indexedAndIndexableCounts = async () => { * (can use {@link addFileEntry} to inform it about new files). From this * universe, we filter out fileIDs the files corresponding to which have already * been indexed, or for which we attempted indexing but failed. + * + * @param count Limit the result to up to {@link count} items. */ -export const unindexedFileIDs = async () => { +export const unindexedFileIDs = async (count?: number) => { const db = await faceDB(); const tx = db.transaction("file-status", "readonly"); - return tx.store.index("isIndexable").getAllKeys(IDBKeyRange.only(1)); + return tx.store.index("isIndexable").getAllKeys(IDBKeyRange.only(1), count); }; /** diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index 7ea723f135..475b2892ee 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -1,6 +1,7 @@ import { FILE_TYPE } from "@/media/file-type"; import log from "@/next/log"; import { ComlinkWorker } from "@/next/worker/comlink-worker"; +import { ensure } from "@/utils/ensure"; import { wait } from "@/utils/promise"; import { type Remote } from "comlink"; import mlIDbStorage, { ML_SEARCH_CONFIG_NAME } from "services/face/db-old"; @@ -11,8 +12,12 @@ import machineLearningService, { import mlWorkManager from "services/machineLearning/mlWorkManager"; import type { EnteFile } from "types/file"; import { isInternalUserForML } from "utils/user"; -import { indexedAndIndexableCounts } from "./db"; -import type { IndexStatus, MinimalPersistedFileData } from "./db-old"; +import { + indexedAndIndexableCounts, + syncWithLocalIndexableFileIDs, + unindexedFileIDs, +} from "./db"; +import type { IndexStatus } from "./db-old"; import { FaceIndexerWorker } from "./indexer.worker"; /** @@ -245,73 +250,18 @@ export const setIsFaceIndexingEnabled = async (enabled: boolean) => { return mlIDbStorage.putConfig(ML_SEARCH_CONFIG_NAME, { enabled }); }; -export const syncLocalFiles = async (userID: number) => { - const localFilesMap = await localIndexableFilesByID(userID); - - // const localFileIDs = new Set(localFilesMap.keys()); - - const db = await mlIDbStorage.db; - const tx = db.transaction("files", "readwrite"); - const mlFileIdsArr = await mlIDbStorage.getAllFileIdsForUpdate(tx); - const mlFileIds = new Set(); - mlFileIdsArr.forEach((mlFileId) => mlFileIds.add(mlFileId)); - - const newFileIds: Array = []; - for (const localFileId of localFilesMap.keys()) { - if (!mlFileIds.has(localFileId)) { - newFileIds.push(localFileId); - } - } - - let updated = false; - if (newFileIds.length > 0) { - log.info("newFiles: ", newFileIds.length); - const newFiles = newFileIds.map( - (fileID) => - ({ - fileID, - mlVersion: 0, - errorCount: 0, - faceEmbedding: { faces: [] }, - }) as MinimalPersistedFileData, - ); - await mlIDbStorage.putAllFiles(newFiles, tx); - updated = true; - } - - const removedFileIds: Array = []; - for (const mlFileId of mlFileIds) { - if (!localFilesMap.has(mlFileId)) { - removedFileIds.push(mlFileId); - } - } - - if (removedFileIds.length > 0) { - log.info("removedFiles: ", removedFileIds.length); - await mlIDbStorage.removeAllFiles(removedFileIds, tx); - updated = true; - } - - await tx.done; - - if (updated) { - // TODO: should do in same transaction - await mlIDbStorage.incrementIndexVersion("files"); - } - - return localFilesMap; -}; - /** - * Return a map of all {@link EnteFile}s owned by {@link userID} that we know - * about locally, indexed by their {@link fileID}. + * Sync face DB with the local indexable files that we know about. Then return + * the next {@link count} files that still need to be indexed. * - * @param userID Restrict the returned files to those owned by a {@link userID}. + * For more specifics of what a "sync" entails, see + * {@link syncWithLocalIndexableFileIDs}. + * + * @param userID Limit indexing to files owned by a {@link userID}. + * + * @param count Limit the resulting list of files to {@link count}. */ -const localIndexableFilesByID = async ( - userID: number, -): Promise> => { - const result = new Map(); +export const getFilesToIndex = async (userID: number, count: number) => { const localFiles = await getLocalFiles(); const indexableTypes = [FILE_TYPE.IMAGE, FILE_TYPE.LIVE_PHOTO]; const indexableFiles = localFiles.filter( @@ -319,6 +269,10 @@ const localIndexableFilesByID = async ( f.ownerID == userID && indexableTypes.includes(f.metadata.fileType), ); - indexableFiles.forEach((f) => result.set(f.id, f)); - return result; + const filesByID = new Map(indexableFiles.map((f) => [f.id, f])); + + await syncWithLocalIndexableFileIDs([...filesByID.keys()]); + + const fileIDsToIndex = await unindexedFileIDs(count); + return fileIDsToIndex.map((id) => ensure(filesByID.get(id))); }; diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index 47517b4e02..35a2be471a 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -2,7 +2,7 @@ import log from "@/next/log"; import { CustomError, parseUploadErrorCodes } from "@ente/shared/error"; import PQueue from "p-queue"; import mlIDbStorage from "services/face/db-old"; -import { syncLocalFiles } from "services/face/indexer"; +import { getFilesToIndex } from "services/face/indexer"; import { FaceIndexerWorker } from "services/face/indexer.worker"; import { EnteFile } from "types/file"; @@ -63,10 +63,7 @@ class MachineLearningService { const syncContext = await this.getSyncContext(token, userID, userAgent); - const localFiles = await syncLocalFiles(userID); - syncContext.localFilesMap = localFiles; - - await this.getOutOfSyncFiles(syncContext); + syncContext.outOfSyncFiles = await getFilesToIndex(userID, batchSize); if (syncContext.outOfSyncFiles.length > 0) { await this.syncFiles(syncContext); @@ -77,23 +74,6 @@ class MachineLearningService { return !error && nOutOfSyncFiles > 0; } - private async getOutOfSyncFiles(syncContext: MLSyncContext) { - const startTime = Date.now(); - const fileIds = await mlIDbStorage.getFileIds( - batchSize, - defaultMLVersion, - MAX_ML_SYNC_ERROR_COUNT, - ); - - log.info("fileIds: ", JSON.stringify(fileIds)); - - const localFilesMap = syncContext.localFilesMap; - syncContext.outOfSyncFiles = fileIds.map((fileId) => - localFilesMap.get(fileId), - ); - log.info("getOutOfSyncFiles", Date.now() - startTime, "ms"); - } - private async syncFiles(syncContext: MLSyncContext) { this.isSyncing = true; try { From 3e1dbce6298a91f28c880b3347f95671e7b545b2 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 18:57:19 +0530 Subject: [PATCH 70/77] Prune --- .../machineLearning/machineLearningService.ts | 16 ++-------------- .../services/machineLearning/mlWorkManager.ts | 5 +++++ 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index 35a2be471a..6612e3c443 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -1,7 +1,6 @@ import log from "@/next/log"; import { CustomError, parseUploadErrorCodes } from "@ente/shared/error"; import PQueue from "p-queue"; -import mlIDbStorage from "services/face/db-old"; import { getFilesToIndex } from "services/face/indexer"; import { FaceIndexerWorker } from "services/face/indexer.worker"; import { EnteFile } from "types/file"; @@ -96,12 +95,6 @@ class MachineLearningService { } await syncContext.syncQueue.onIdle(); this.isSyncing = false; - - // TODO: In case syncJob has to use multiple ml workers - // do in same transaction with each file update - // or keep in files store itself - await mlIDbStorage.incrementIndexVersion("files"); - // await this.disposeMLModels(); } private async getSyncContext( @@ -208,17 +201,12 @@ class MachineLearningService { private async syncFile( enteFile: EnteFile, - localFile: globalThis.File | undefined, + file: File | undefined, userAgent: string, ) { - const oldMlFile = await mlIDbStorage.getFile(enteFile.id); - if (oldMlFile && oldMlFile.mlVersion) { - return; - } - const worker = new FaceIndexerWorker(); - await worker.index(enteFile, localFile, userAgent); + await worker.index(enteFile, file, userAgent); } } diff --git a/web/apps/photos/src/services/machineLearning/mlWorkManager.ts b/web/apps/photos/src/services/machineLearning/mlWorkManager.ts index 3f7e9571e8..2d58efc32c 100644 --- a/web/apps/photos/src/services/machineLearning/mlWorkManager.ts +++ b/web/apps/photos/src/services/machineLearning/mlWorkManager.ts @@ -228,7 +228,11 @@ class MLWorkManager { this.mlSearchEnabled && this.startSyncJob(); } + // eslint-disable-next-line @typescript-eslint/no-unused-vars public async syncLocalFile(enteFile: EnteFile, localFile: globalThis.File) { + return; + /* + TODO-ML(MR): Disable live sync for now await this.liveSyncQueue.add(async () => { this.stopSyncJob(); const token = getToken(); @@ -243,6 +247,7 @@ class MLWorkManager { localFile, ); }); + */ } // Sync Job From 654f6b8934568f4f63c2f9dfa90212dd5a499d14 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 19:04:01 +0530 Subject: [PATCH 71/77] Remove old indexstatus --- web/apps/photos/src/services/face/indexer.ts | 32 ++------------------ 1 file changed, 2 insertions(+), 30 deletions(-) diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index 475b2892ee..2ce75bf008 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -1,14 +1,11 @@ import { FILE_TYPE } from "@/media/file-type"; -import log from "@/next/log"; import { ComlinkWorker } from "@/next/worker/comlink-worker"; import { ensure } from "@/utils/ensure"; import { wait } from "@/utils/promise"; import { type Remote } from "comlink"; import mlIDbStorage, { ML_SEARCH_CONFIG_NAME } from "services/face/db-old"; import { getLocalFiles } from "services/fileService"; -import machineLearningService, { - defaultMLVersion, -} from "services/machineLearning/machineLearningService"; +import machineLearningService from "services/machineLearning/machineLearningService"; import mlWorkManager from "services/machineLearning/mlWorkManager"; import type { EnteFile } from "types/file"; import { isInternalUserForML } from "utils/user"; @@ -17,7 +14,6 @@ import { syncWithLocalIndexableFileIDs, unindexedFileIDs, } from "./db"; -import type { IndexStatus } from "./db-old"; import { FaceIndexerWorker } from "./indexer.worker"; /** @@ -180,35 +176,11 @@ export const faceIndexingStatus = async (): Promise => { phase = "done"; } - const indexingStatus = { + return { phase, nSyncedFiles: indexedCount, nTotalFiles: indexableCount, }; - - const indexStatus0 = await mlIDbStorage.getIndexStatus(defaultMLVersion); - const indexStatus = convertToNewInterface(indexStatus0); - - log.debug(() => ({ indexStatus, indexingStatus })); - - return indexStatus; -}; - -const convertToNewInterface = (indexStatus: IndexStatus) => { - let phase: FaceIndexingStatus["phase"]; - if (!indexStatus.localFilesSynced) { - phase = "scheduled"; - } else if (indexStatus.outOfSyncFilesExists) { - phase = "indexing"; - } else if (!indexStatus.peopleIndexSynced) { - phase = "clustering"; - } else { - phase = "done"; - } - return { - ...indexStatus, - phase, - }; }; /** From 7cc29c302e8d2cdff3764c8484488eb7848e69f6 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 19:11:44 +0530 Subject: [PATCH 72/77] new --- web/apps/photos/src/services/face/db.ts | 8 ++++++++ web/apps/photos/src/services/face/indexer.ts | 5 +++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/web/apps/photos/src/services/face/db.ts b/web/apps/photos/src/services/face/db.ts index 21dd3de41d..afc9581382 100644 --- a/web/apps/photos/src/services/face/db.ts +++ b/web/apps/photos/src/services/face/db.ts @@ -173,6 +173,14 @@ export const saveFaceIndex = async (faceIndex: FaceIndex) => { ]); }; +/** + * Return the {@link FaceIndex}, if any, for {@link fileID}. + */ +export const faceIndex = async (fileID: number) => { + const db = await faceDB(); + return db.get("face-index", fileID); +}; + /** * Record the existence of a file so that entities in the face indexing universe * know about it (e.g. can index it if it is new and it needs indexing). diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index 2ce75bf008..cb8465d166 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -10,6 +10,7 @@ import mlWorkManager from "services/machineLearning/mlWorkManager"; import type { EnteFile } from "types/file"; import { isInternalUserForML } from "utils/user"; import { + faceIndex, indexedAndIndexableCounts, syncWithLocalIndexableFileIDs, unindexedFileIDs, @@ -190,8 +191,8 @@ export const faceIndexingStatus = async (): Promise => { export const unidentifiedFaceIDs = async ( enteFile: EnteFile, ): Promise => { - const mlFileData = await mlIDbStorage.getFile(enteFile.id); - return mlFileData?.faceEmbedding.faces.map((f) => f.faceID) ?? []; + const index = await faceIndex(enteFile.id); + return index?.faceEmbedding.faces.map((f) => f.faceID) ?? []; }; /** From 400a6a9054ba7510ad6ef66a1cc97631681663b3 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 19:17:24 +0530 Subject: [PATCH 73/77] Store enabled state in local storage --- web/apps/photos/src/services/face/indexer.ts | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index cb8465d166..e2b62e51d0 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -3,7 +3,6 @@ import { ComlinkWorker } from "@/next/worker/comlink-worker"; import { ensure } from "@/utils/ensure"; import { wait } from "@/utils/promise"; import { type Remote } from "comlink"; -import mlIDbStorage, { ML_SEARCH_CONFIG_NAME } from "services/face/db-old"; import { getLocalFiles } from "services/fileService"; import machineLearningService from "services/machineLearning/machineLearningService"; import mlWorkManager from "services/machineLearning/mlWorkManager"; @@ -206,10 +205,7 @@ export const unidentifiedFaceIDs = async ( */ export const isFaceIndexingEnabled = async () => { if (isInternalUserForML()) { - const config = await mlIDbStorage.getConfig(ML_SEARCH_CONFIG_NAME, { - enabled: false, - }); - return config.enabled; + return localStorage.getItem("faceIndexingEnabled") == "1"; } // Force disabled for everyone else while we finalize it to avoid redundant // reindexing for users. @@ -220,7 +216,8 @@ export const isFaceIndexingEnabled = async () => { * Update the (locally stored) value of {@link isFaceIndexingEnabled}. */ export const setIsFaceIndexingEnabled = async (enabled: boolean) => { - return mlIDbStorage.putConfig(ML_SEARCH_CONFIG_NAME, { enabled }); + if (enabled) localStorage.setItem("faceIndexingEnabled", "1"); + else localStorage.removeItem("faceIndexingEnabled"); }; /** From 21567d546e7c049e4031fc838bb0fbb41208b377 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 19:21:38 +0530 Subject: [PATCH 74/77] bye --- web/apps/photos/src/services/face/db-old.ts | 391 ------------------ web/apps/photos/src/services/face/db.ts | 10 + .../photos/src/services/face/types-old.ts | 6 - .../machineLearning/machineLearningService.ts | 2 - .../services/machineLearning/mlWorkManager.ts | 2 - 5 files changed, 10 insertions(+), 401 deletions(-) delete mode 100644 web/apps/photos/src/services/face/db-old.ts delete mode 100644 web/apps/photos/src/services/face/types-old.ts diff --git a/web/apps/photos/src/services/face/db-old.ts b/web/apps/photos/src/services/face/db-old.ts deleted file mode 100644 index b66c72d811..0000000000 --- a/web/apps/photos/src/services/face/db-old.ts +++ /dev/null @@ -1,391 +0,0 @@ -import { haveWindow } from "@/next/env"; -import log from "@/next/log"; -import { - DBSchema, - IDBPDatabase, - IDBPTransaction, - StoreNames, - deleteDB, - openDB, -} from "idb"; -import isElectron from "is-electron"; -import type { Person } from "services/face/people"; -import type { MlFileData } from "services/face/types-old"; -import { MAX_ML_SYNC_ERROR_COUNT } from "services/machineLearning/machineLearningService"; - -export interface IndexStatus { - outOfSyncFilesExists: boolean; - nSyncedFiles: number; - nTotalFiles: number; - localFilesSynced: boolean; - peopleIndexSynced: boolean; -} - -/** - * TODO(MR): Transient type with an intersection of values that both existing - * and new types during the migration will have. Eventually we'll store the the - * server ML data shape here exactly. - */ -export interface MinimalPersistedFileData { - fileID: number; - mlVersion: number; - errorCount: number; - faceEmbedding: { faces: { faceID: string }[] }; -} - -interface Config {} - -export const ML_SEARCH_CONFIG_NAME = "ml-search"; - -const MLDATA_DB_NAME = "mldata"; -interface MLDb extends DBSchema { - files: { - key: number; - value: MinimalPersistedFileData; - indexes: { mlVersion: [number, number] }; - }; - people: { - key: number; - value: Person; - }; - // Unused, we only retain this is the schema so that we can delete it during - // migration. - things: { - key: number; - value: unknown; - }; - versions: { - key: string; - value: number; - }; - library: { - key: string; - value: unknown; - }; - configs: { - key: string; - value: Config; - }; -} - -class MLIDbStorage { - public _db: Promise>; - - constructor() { - if (!haveWindow() || !isElectron()) { - return; - } - - this.db; - } - - private openDB(): Promise> { - return openDB(MLDATA_DB_NAME, 4, { - terminated: async () => { - log.error("ML Indexed DB terminated"); - this._db = undefined; - // TODO: remove if there is chance of this going into recursion in some case - await this.db; - }, - blocked() { - // TODO: make sure we dont allow multiple tabs of app - log.error("ML Indexed DB blocked"); - }, - blocking() { - // TODO: make sure we dont allow multiple tabs of app - log.error("ML Indexed DB blocking"); - }, - async upgrade(db, oldVersion, newVersion, tx) { - let wasMLSearchEnabled = false; - try { - const searchConfig: unknown = await tx - .objectStore("configs") - .get(ML_SEARCH_CONFIG_NAME); - if ( - searchConfig && - typeof searchConfig == "object" && - "enabled" in searchConfig && - typeof searchConfig.enabled == "boolean" - ) { - wasMLSearchEnabled = searchConfig.enabled; - } - } catch (e) { - // The configs store might not exist (e.g. during logout). - // Ignore. - } - log.info( - `Previous ML database v${oldVersion} had ML search ${wasMLSearchEnabled ? "enabled" : "disabled"}`, - ); - - if (oldVersion < 1) { - const filesStore = db.createObjectStore("files", { - // keyPath: "fileId", TODO(MR): Changing this, since - // we're going to be deleting this DB before this PR is - // merged. - keyPath: "fileID", - }); - filesStore.createIndex("mlVersion", [ - "mlVersion", - "errorCount", - ]); - - db.createObjectStore("people", { - keyPath: "id", - }); - - db.createObjectStore("things", { - keyPath: "id", - }); - - db.createObjectStore("versions"); - - db.createObjectStore("library"); - } - if (oldVersion < 2) { - // TODO: update configs if version is updated in defaults - db.createObjectStore("configs"); - - /* - await tx - .objectStore("configs") - .add( - DEFAULT_ML_SYNC_JOB_CONFIG, - "ml-sync-job", - ); - - await tx - .objectStore("configs") - .add(DEFAULT_ML_SYNC_CONFIG, ML_SYNC_CONFIG_NAME); - */ - } - if (oldVersion < 3) { - const DEFAULT_ML_SEARCH_CONFIG = { - enabled: false, - }; - - await tx - .objectStore("configs") - .add(DEFAULT_ML_SEARCH_CONFIG, ML_SEARCH_CONFIG_NAME); - } - /* - This'll go in version 5. Note that version 4 was never released, - but it was in main for a while, so we'll just skip it to avoid - breaking the upgrade path for people who ran the mainline. - */ - if (oldVersion < 4) { - /* - try { - await tx - .objectStore("configs") - .delete(ML_SEARCH_CONFIG_NAME); - - await tx - .objectStore("configs") - .delete(""ml-sync""); - - await tx - .objectStore("configs") - .delete("ml-sync-job"); - - await tx - .objectStore("configs") - .add( - { enabled: wasMLSearchEnabled }, - ML_SEARCH_CONFIG_NAME, - ); - - db.deleteObjectStore("library"); - db.deleteObjectStore("things"); - } catch { - // TODO: ignore for now as we finalize the new version - // the shipped implementation should have a more - // deterministic migration. - } - */ - } - log.info( - `ML DB upgraded from version ${oldVersion} to version ${newVersion}`, - ); - }, - }); - } - - public get db(): Promise> { - if (!this._db) { - this._db = this.openDB(); - log.info("Opening Ml DB"); - } - - return this._db; - } - - public async clearMLDB() { - const db = await this.db; - db.close(); - await deleteDB(MLDATA_DB_NAME); - log.info("Cleared Ml DB"); - this._db = undefined; - await this.db; - } - - public async getAllFileIdsForUpdate( - tx: IDBPTransaction, - ) { - return tx.store.getAllKeys(); - } - - public async getFileIds( - count: number, - limitMlVersion: number, - maxErrorCount: number, - ) { - const db = await this.db; - const tx = db.transaction("files", "readonly"); - const index = tx.store.index("mlVersion"); - let cursor = await index.openKeyCursor( - IDBKeyRange.upperBound([limitMlVersion], true), - ); - - const fileIds: number[] = []; - while (cursor && fileIds.length < count) { - if ( - cursor.key[0] < limitMlVersion && - cursor.key[1] <= maxErrorCount - ) { - fileIds.push(cursor.primaryKey); - } - cursor = await cursor.continue(); - } - await tx.done; - - return fileIds; - } - - public async getFile(fileId: number): Promise { - const db = await this.db; - return db.get("files", fileId); - } - - public async putFile(mlFile: MlFileData) { - const db = await this.db; - return db.put("files", mlFile); - } - - public async upsertFileInTx( - fileId: number, - upsert: (mlFile: MinimalPersistedFileData) => MinimalPersistedFileData, - ) { - const db = await this.db; - const tx = db.transaction("files", "readwrite"); - const existing = await tx.store.get(fileId); - const updated = upsert(existing); - await tx.store.put(updated); - await tx.done; - - return updated; - } - - public async putAllFiles( - mlFiles: MinimalPersistedFileData[], - tx: IDBPTransaction, - ) { - await Promise.all(mlFiles.map((mlFile) => tx.store.put(mlFile))); - } - - public async removeAllFiles( - fileIds: Array, - tx: IDBPTransaction, - ) { - await Promise.all(fileIds.map((fileId) => tx.store.delete(fileId))); - } - - public async getPerson(id: number) { - const db = await this.db; - return db.get("people", id); - } - - public async getAllPeople() { - const db = await this.db; - return db.getAll("people"); - } - - public async incrementIndexVersion(index: StoreNames) { - if (index === "versions") { - throw new Error("versions store can not be versioned"); - } - const db = await this.db; - const tx = db.transaction(["versions", index], "readwrite"); - let version = await tx.objectStore("versions").get(index); - version = (version || 0) + 1; - tx.objectStore("versions").put(version, index); - await tx.done; - - return version; - } - - public async getConfig(name: string, def: T) { - const db = await this.db; - const tx = db.transaction("configs", "readwrite"); - let config = (await tx.store.get(name)) as T; - if (!config) { - config = def; - await tx.store.put(def, name); - } - await tx.done; - - return config; - } - - public async putConfig(name: string, data: Config) { - const db = await this.db; - return db.put("configs", data, name); - } - - public async getIndexStatus(latestMlVersion: number): Promise { - const db = await this.db; - const tx = db.transaction(["files", "versions"], "readonly"); - const mlVersionIdx = tx.objectStore("files").index("mlVersion"); - - let outOfSyncCursor = await mlVersionIdx.openKeyCursor( - IDBKeyRange.upperBound([latestMlVersion], true), - ); - let outOfSyncFilesExists = false; - while (outOfSyncCursor && !outOfSyncFilesExists) { - if ( - outOfSyncCursor.key[0] < latestMlVersion && - outOfSyncCursor.key[1] <= MAX_ML_SYNC_ERROR_COUNT - ) { - outOfSyncFilesExists = true; - } - outOfSyncCursor = await outOfSyncCursor.continue(); - } - - const nSyncedFiles = await mlVersionIdx.count( - IDBKeyRange.lowerBound([latestMlVersion]), - ); - const nTotalFiles = await mlVersionIdx.count(); - - const filesIndexVersion = await tx.objectStore("versions").get("files"); - const peopleIndexVersion = await tx - .objectStore("versions") - .get("people"); - const filesIndexVersionExists = - filesIndexVersion !== null && filesIndexVersion !== undefined; - const peopleIndexVersionExists = - peopleIndexVersion !== null && peopleIndexVersion !== undefined; - - await tx.done; - - return { - outOfSyncFilesExists, - nSyncedFiles, - nTotalFiles, - localFilesSynced: filesIndexVersionExists, - peopleIndexSynced: - peopleIndexVersionExists && - peopleIndexVersion === filesIndexVersion, - }; - } -} - -export default new MLIDbStorage(); diff --git a/web/apps/photos/src/services/face/db.ts b/web/apps/photos/src/services/face/db.ts index afc9581382..ab03b726f7 100644 --- a/web/apps/photos/src/services/face/db.ts +++ b/web/apps/photos/src/services/face/db.ts @@ -82,6 +82,8 @@ interface FileStatus { let _faceDB: ReturnType | undefined; const openFaceDB = async () => { + deleteLegacyDB(); + const db = await openDB("face", 1, { upgrade(db, oldVersion, newVersion) { log.info(`Upgrading face DB ${oldVersion} => ${newVersion}`); @@ -112,6 +114,13 @@ const openFaceDB = async () => { return db; }; +const deleteLegacyDB = () => { + // Delete the legacy face DB. + // This code was added June 2024 (v1.7.1-rc) and can be removed once clients + // have migrated over. + void deleteDB("mldata"); +}; + /** * @returns a lazily created, cached connection to the face DB. */ @@ -138,6 +147,7 @@ export const closeFaceDBConnectionsIfNeeded = async () => { * Meant to be called during logout. */ export const clearFaceData = async () => { + deleteLegacyDB(); await closeFaceDBConnectionsIfNeeded(); return deleteDB("face", { blocked() { diff --git a/web/apps/photos/src/services/face/types-old.ts b/web/apps/photos/src/services/face/types-old.ts deleted file mode 100644 index d2bf61a4f8..0000000000 --- a/web/apps/photos/src/services/face/types-old.ts +++ /dev/null @@ -1,6 +0,0 @@ -import type { FaceIndex } from "./types"; - -export type MlFileData = FaceIndex & { - mlVersion: number; - errorCount: number; -}; diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index 6612e3c443..196f3d46e8 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -9,8 +9,6 @@ export const defaultMLVersion = 1; const batchSize = 200; -export const MAX_ML_SYNC_ERROR_COUNT = 1; - class MLSyncContext { public token: string; public userID: number; diff --git a/web/apps/photos/src/services/machineLearning/mlWorkManager.ts b/web/apps/photos/src/services/machineLearning/mlWorkManager.ts index 2d58efc32c..9502c5a75d 100644 --- a/web/apps/photos/src/services/machineLearning/mlWorkManager.ts +++ b/web/apps/photos/src/services/machineLearning/mlWorkManager.ts @@ -8,7 +8,6 @@ import { getToken, getUserID } from "@ente/shared/storage/localStorage/helpers"; import debounce from "debounce"; import PQueue from "p-queue"; import { createFaceComlinkWorker } from "services/face"; -import mlIDbStorage from "services/face/db-old"; import type { DedicatedMLWorker } from "services/face/face.worker"; import { EnteFile } from "types/file"; @@ -167,7 +166,6 @@ class MLWorkManager { this.stopSyncJob(); this.mlSyncJob = undefined; await this.terminateLiveSyncWorker(); - await mlIDbStorage.clearMLDB(); } private async fileUploadedHandler(arg: { From 8f7af989bb310b77efc21d5e19ee44b20e22c55b Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 19:30:03 +0530 Subject: [PATCH 75/77] Remove unused --- web/apps/photos/src/services/face/f-index.ts | 3 --- .../src/services/machineLearning/machineLearningService.ts | 2 -- 2 files changed, 5 deletions(-) diff --git a/web/apps/photos/src/services/face/f-index.ts b/web/apps/photos/src/services/face/f-index.ts index a0faaea34b..79a428d0bc 100644 --- a/web/apps/photos/src/services/face/f-index.ts +++ b/web/apps/photos/src/services/face/f-index.ts @@ -4,7 +4,6 @@ import log from "@/next/log"; import { workerBridge } from "@/next/worker/worker-bridge"; import { Matrix } from "ml-matrix"; import DownloadManager from "services/download"; -import { defaultMLVersion } from "services/machineLearning/machineLearningService"; import { getSimilarityTransformation } from "similarity-transformation"; import { Matrix as TransformationMatrix, @@ -70,8 +69,6 @@ export const indexFaces = async ( client: userAgent, faces: await indexFacesInBitmap(fileID, imageBitmap), }, - mlVersion: defaultMLVersion, - errorCount: 0, }; } finally { imageBitmap.close(); diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index 196f3d46e8..8549ec7655 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -5,8 +5,6 @@ import { getFilesToIndex } from "services/face/indexer"; import { FaceIndexerWorker } from "services/face/indexer.worker"; import { EnteFile } from "types/file"; -export const defaultMLVersion = 1; - const batchSize = 200; class MLSyncContext { From f647355666c7b0701caa5ee7934ffdd66b73e955 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 20:41:53 +0530 Subject: [PATCH 76/77] [desktop] Nightly builds --- desktop/.github/workflows/desktop-release.yml | 22 ++++++-- desktop/docs/release.md | 56 ++++++++----------- 2 files changed, 41 insertions(+), 37 deletions(-) diff --git a/desktop/.github/workflows/desktop-release.yml b/desktop/.github/workflows/desktop-release.yml index 70eedf3ea6..e9457240aa 100644 --- a/desktop/.github/workflows/desktop-release.yml +++ b/desktop/.github/workflows/desktop-release.yml @@ -5,12 +5,20 @@ name: "Release" # For more details, see `docs/release.md` in ente-io/ente. on: - # Trigger manually or `gh workflow run desktop-release.yml`. + # Trigger manually or `gh workflow run desktop-release.yml --source=foo`. workflow_dispatch: + inputs: + source: + description: "Branch (ente-io/ente) to build" + required: true + type: string + schedule: + # Run everyday at ~8:00 AM IST (except Sundays). + # See: [Note: Run workflow every 24 hours] + # + - cron: "45 2 * * 1-6" push: # Run when a tag matching the pattern "v*"" is pushed. - # - # See: [Note: Testing release workflows that are triggered by tags]. tags: - "v*" @@ -30,9 +38,13 @@ jobs: - name: Checkout code uses: actions/checkout@v4 with: - # Checkout the desktop/rc branch from the source repository. + # If triggered by a tag, checkout photosd-$tag from the source + # repository. Otherwise checkout $source (default: "main"). repository: ente-io/ente - ref: desktop/rc + ref: + "${{ startsWith(github.ref, 'refs/tags/v') && + format('photosd-{0}', github.ref_name) || ( inputs.source + || 'main' ) }}" submodules: recursive - name: Setup node diff --git a/desktop/docs/release.md b/desktop/docs/release.md index 53d0355c3e..f56d2c4404 100644 --- a/desktop/docs/release.md +++ b/desktop/docs/release.md @@ -5,7 +5,7 @@ creates a draft release with artifacts built. When ready, we publish that release. The download links on our website, and existing apps already check the latest GitHub release and update accordingly. -The complication comes by the fact that electron-builder's auto updaterr (the +The complication comes by the fact that electron-builder's auto updater (the mechanism that we use for auto updates) doesn't work with monorepos. So we need to keep a separate (non-mono) repository just for doing releases. @@ -16,48 +16,45 @@ to keep a separate (non-mono) repository just for doing releases. ## Workflow - Release Candidates -Leading up to the release, we can make one or more draft releases that are not -intended to be published, but serve as test release candidates. +Nightly RC builds of `main` are published by a scheduled workflow automatically. +If needed, these builds can also be manually triggered, including specifying the +source repository branch to build: -The workflow for making such "rc" builds is: +```sh +gh workflow run desktop-release.yml --source= +``` -1. Update `package.json` in the source repo to use version `1.x.x-rc`. Create a - new draft release in the release repo with title `1.x.x-rc`. In the tag - input enter `v1.x.x-rc` and select the option to "create a new tag on - publish". - -2. Push code to the `desktop/rc` branch in the source repo. - -3. Trigger the GitHub action in the release repo - - ```sh - gh workflow run desktop-release.yml - ``` - -We can do steps 2 and 3 multiple times: each time it'll just update the -artifacts attached to the same draft. +Each such workflow run will update the artifacts attached to the same +(pre-existing) pre-release. ## Workflow - Release 1. Update source repo to set version `1.x.x` in `package.json` and finalize the CHANGELOG. -2. Push code to the `desktop/rc` branch in the source repo. +2. Merge PR then tag the merge commit on `main` in the source repo: -3. In the release repo + ```sh + git tag photosd-v1.x.x + git push origin photosd-v1.x.x + ``` + +3. In the release repo: ```sh ./.github/trigger-release.sh v1.x.x ``` -4. If the build is successful, tag `desktop/rc` in the source repo. +This'll trigger the workflow and create a new draft release, which you can +publish after adding the release notes. - ```sh - # Assuming we're on desktop/rc that just got built +The release is done at this point, and we can now start a new RC train for +subsequent nightly builds. - git tag photosd-v1.x.x - git push origin photosd-v1.x.x - ``` +1. Update `package.json` in the source repo to use version `1.x.x-rc`. Create a + new draft release in the release repo with title `1.x.x-rc`. In the tag + input enter `v1.x.x-rc` and select the option to "create a new tag on + publish". ## Post build @@ -87,8 +84,3 @@ everything is automated: now their maintainers automatically bump the SHA, version number and the (derived from the version) URL in the formula when their tools notice a new release on our GitHub. - -We can also publish the draft releases by checking the "pre-release" option. -Such releases don't cause any of the channels (our website, or the desktop app -auto updater, or brew) to be notified, instead these are useful for giving links -to pre-release builds to customers. From c1097de27f6f09e1f9622616a340b235ed596c67 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 May 2024 21:02:17 +0530 Subject: [PATCH 77/77] Non required --- desktop/.github/workflows/desktop-release.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/desktop/.github/workflows/desktop-release.yml b/desktop/.github/workflows/desktop-release.yml index e9457240aa..a5d7b9e155 100644 --- a/desktop/.github/workflows/desktop-release.yml +++ b/desktop/.github/workflows/desktop-release.yml @@ -10,7 +10,6 @@ on: inputs: source: description: "Branch (ente-io/ente) to build" - required: true type: string schedule: # Run everyday at ~8:00 AM IST (except Sundays).