From 0943d1db8cf6a012389148892f0a6a2c693f28ae Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 26 Jun 2024 18:22:54 +0530 Subject: [PATCH 01/34] wip doc --- web/packages/new/photos/services/embedding.ts | 48 +++++++++++++++---- 1 file changed, 40 insertions(+), 8 deletions(-) diff --git a/web/packages/new/photos/services/embedding.ts b/web/packages/new/photos/services/embedding.ts index 9492a47e11..219747e826 100644 --- a/web/packages/new/photos/services/embedding.ts +++ b/web/packages/new/photos/services/embedding.ts @@ -1,9 +1,9 @@ import { authenticatedRequestHeaders } from "@/next/http"; import { apiURL } from "@/next/origins"; import { nullToUndefined } from "@/utils/transform"; -// import ComlinkCryptoWorker from "@ente/shared/crypto"; +import ComlinkCryptoWorker from "@ente/shared/crypto"; import { z } from "zod"; -// import { getAllLocalFiles } from "./files"; +import { getAllLocalFiles } from "./files"; /** * The embeddings that we (the current client) knows how to handle. @@ -74,15 +74,47 @@ type RemoteEmbedding = z.infer; /** * Ask remote for what all changes have happened to the face embeddings that it - * knows about since the last time we synced. Then update our local state to - * reflect those changes. + * knows about since the last time we synced. Update our local state to reflect + * those changes. * * It takes no parameters since it saves the last sync time in local storage. + * + * Precondition: This function should be called only after we have synced files + * with remote (See: [Note: Ignoring embeddings for unknown files]). */ export const syncRemoteFaceEmbeddings = async () => { let sinceTime = faceEmbeddingSyncTime(); - // const cryptoWorker = await ComlinkCryptoWorker.getInstance(); - // const files = await getAllLocalFiles(); + const cryptoWorker = await ComlinkCryptoWorker.getInstance(); + const localFiles = await getAllLocalFiles(); + const localFilesByID = new Map(localFiles.map((f) => [f.id, f])); + + const decryptEmbedding = async (remoteEmbedding: RemoteEmbedding) => { + const file = localFilesByID.get(remoteEmbedding.fileID) + // [Note: Ignoring embeddings for unknown files] + // + // We need the file to decrypt the embedding. This is easily ensured by + // running the embedding sync after we have synced our local files with + // remote. + // + // Still, it might happen that we come across an embedding for which we + // don't have the corresponding file locally. We can put them in two + // buckets: + // + // 1. Known case: In rare cases we might get a diff entry for an + // embedding corresponding to a file which has been deleted (but + // whose embedding is enqueued for deletion). Client should expect + // such a scenario, but all they have to do is just ignore such + // embeddings. + // + // 2. Other unknown cases: Even if somehow we end up with an embedding + // for a existent file which we don't have locally, it is fine + // because the current client will just regenerate the embedding if + // the file really exists and gets locally found later. There would + // be a bit of duplicate work, but that's fine as long as there + // isn't a systematic scenario where this happens. + if (!file) return undefined; + + } // TODO: eslint has fixed this spurious warning, but we're not on the latest // version yet, so add a disable. @@ -96,7 +128,7 @@ export const syncRemoteFaceEmbeddings = async () => { ); if (remoteEmbeddings.length == 0) break; // const _embeddings = Promise.all( - // remoteEmbeddings.map(decryptFaceEmbedding), + // remoteEmbeddings.map(decryptEmbedding), // ); sinceTime = remoteEmbeddings.reduce( (max, { updatedAt }) => Math.max(max, updatedAt), @@ -106,7 +138,7 @@ export const syncRemoteFaceEmbeddings = async () => { } }; -// const decryptFaceEmbedding = async (remoteEmbedding: RemoteEmbedding) => { +const decryptFaceEmbedding = async (remoteEmbedding: RemoteEmbedding) => { // const fileKey = fileIdToKeyMap.get(embedding.fileID); // if (!fileKey) { // throw Error(CustomError.FILE_NOT_FOUND); From c4b93019d540dc4decc68c6e53465113a05dd34b Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 26 Jun 2024 18:37:59 +0530 Subject: [PATCH 02/34] doc 2 --- web/packages/new/photos/services/embedding.ts | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/web/packages/new/photos/services/embedding.ts b/web/packages/new/photos/services/embedding.ts index 219747e826..5ec03a7319 100644 --- a/web/packages/new/photos/services/embedding.ts +++ b/web/packages/new/photos/services/embedding.ts @@ -172,7 +172,39 @@ const saveFaceEmbeddingSyncTime = (t: number) => // } -/** The maximum number of items to fetch in a single GET /embeddings/diff */ +/** + * The maximum number of items to fetch in a single GET /embeddings/diff + * + * [Note: Limit of returned items in /diff requests] + * + * The various GET /diff API methods, which tell the client what all has changed + * since a timestamp (provided by the client) take a limit parameter. + * + * These diff API calls return all items whose updated at is greater + * (non-inclusive) than the timestamp we provide. So there is no mechanism for + * pagination of items which have the same exact updated at. Conceptually, it + * may happen that there are more items than the limit we've provided. + * + * The behaviour of this limit is different for file diff and embeddings diff. + * + * - For file diff, the limit is advisory, and remote may return less, equal + * or more items than the provided limit. The scenario where it returns more + * is when more files than the limit have the same updated at. Theoretically + * it would make the diff response unbounded, however in practice file + * modifications themselves are all batched. Even if the user selects all + * the files in their library and updates them all in one go in the UI, + * their client app must use batched API calls to make those updates, and + * each of those batches would get distinct updated at. + * + * - For embeddings diff, there are no bulk updates and this limit is enforced + * as a maximum. While theoretically it is possible for an arbitrary number + * of files to have the same updated at, in practice it is not possible with + * the current set of APIs where clients PUT individual embeddings (the + * updated at is a server timestamp). And even if somehow a large number of + * files get the same updated at and thus get truncated in the response, it + * won't lead to any data loss, the client which requested that particular + * truncated diff will just regenerate them. + */ const diffLimit = 500; /** @@ -188,6 +220,8 @@ const diffLimit = 500; * * @returns an array of {@link RemoteEmbedding}. The returned array is limited * to a maximum count of {@link diffLimit}. + * + * > See [Note: Limit of returned items in /diff requests]. */ const getEmbeddingsDiff = async ( model: EmbeddingModel, From d5d7efd657c715be8d42c0a6e26807ef95bcdbae Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 26 Jun 2024 19:03:54 +0530 Subject: [PATCH 03/34] doc 3 --- web/packages/new/photos/services/embedding.ts | 5 ++--- web/packages/shared/crypto/internal/crypto.worker.ts | 1 + web/packages/shared/crypto/internal/libsodium.ts | 8 ++++++++ 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/web/packages/new/photos/services/embedding.ts b/web/packages/new/photos/services/embedding.ts index 5ec03a7319..5b422eaff1 100644 --- a/web/packages/new/photos/services/embedding.ts +++ b/web/packages/new/photos/services/embedding.ts @@ -73,9 +73,8 @@ const RemoteEmbedding = z.object({ type RemoteEmbedding = z.infer; /** - * Ask remote for what all changes have happened to the face embeddings that it - * knows about since the last time we synced. Update our local state to reflect - * those changes. + * Fetch new or updated face embeddings with the server and save them locally. + * Also prune local embeddings for any files no longer exist locally. * * It takes no parameters since it saves the last sync time in local storage. * diff --git a/web/packages/shared/crypto/internal/crypto.worker.ts b/web/packages/shared/crypto/internal/crypto.worker.ts index ef4f6a062e..e709aa74d6 100644 --- a/web/packages/shared/crypto/internal/crypto.worker.ts +++ b/web/packages/shared/crypto/internal/crypto.worker.ts @@ -4,6 +4,7 @@ import type { StateAddress } from "libsodium-wrappers"; const textDecoder = new TextDecoder(); const textEncoder = new TextEncoder(); + export class DedicatedCryptoWorker { async decryptMetadata( encryptedMetadata: string, diff --git a/web/packages/shared/crypto/internal/libsodium.ts b/web/packages/shared/crypto/internal/libsodium.ts index 56094c9b2d..9906c1a858 100644 --- a/web/packages/shared/crypto/internal/libsodium.ts +++ b/web/packages/shared/crypto/internal/libsodium.ts @@ -1,3 +1,11 @@ +/** + * @file A thin-ish layer over the actual libsodium APIs, to make them more + * palatable to the rest of our Javascript code. + * + * All functions are stateless, async, and safe to use in Web Workers. + * + * Docs for the JS library: https://github.com/jedisct1/libsodium.js + */ import { mergeUint8Arrays } from "@/utils/array"; import { CustomError } from "@ente/shared/error"; import sodium, { type StateAddress } from "libsodium-wrappers"; From 2f7923b097af3652803a8397fdb4a21db9b3bd06 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 26 Jun 2024 19:04:56 +0530 Subject: [PATCH 04/34] Rearrange --- .../shared/crypto/internal/libsodium.ts | 170 +++++++++--------- 1 file changed, 85 insertions(+), 85 deletions(-) diff --git a/web/packages/shared/crypto/internal/libsodium.ts b/web/packages/shared/crypto/internal/libsodium.ts index 9906c1a858..8c64bc4b0d 100644 --- a/web/packages/shared/crypto/internal/libsodium.ts +++ b/web/packages/shared/crypto/internal/libsodium.ts @@ -12,6 +12,91 @@ import sodium, { type StateAddress } from "libsodium-wrappers"; import { ENCRYPTION_CHUNK_SIZE } from "../constants"; import type { B64EncryptionResult } from "../types"; +export async function fromB64(input: string) { + await sodium.ready; + return sodium.from_base64(input, sodium.base64_variants.ORIGINAL); +} + +export async function toB64(input: Uint8Array) { + await sodium.ready; + return sodium.to_base64(input, sodium.base64_variants.ORIGINAL); +} + +/** Convert a {@link Uint8Array} to a URL safe Base64 encoded string. */ +export const toB64URLSafe = async (input: Uint8Array) => { + await sodium.ready; + return sodium.to_base64(input, sodium.base64_variants.URLSAFE); +}; + +/** + * Convert a {@link Uint8Array} to a URL safe Base64 encoded string. + * + * This differs from {@link toB64URLSafe} in that it does not append any + * trailing padding character(s) "=" to make the resultant string's length be an + * integer multiple of 4. + * + * - In some contexts, for example when serializing WebAuthn binary for + * transmission over the network, this is the required / recommended + * approach. + * + * - In other cases, for example when trying to pass an arbitrary JSON string + * via a URL parameter, this is also convenient so that we do not have to + * deal with any ambiguity surrounding the "=" which is also the query + * parameter key value separator. + */ +export const toB64URLSafeNoPadding = async (input: Uint8Array) => { + await sodium.ready; + return sodium.to_base64(input, sodium.base64_variants.URLSAFE_NO_PADDING); +}; + +/** + * Convert a Base64 encoded string to a {@link Uint8Array}. + * + * This is the converse of {@link toB64URLSafeNoPadding}, and does not expect + * its input string's length to be a an integer multiple of 4. + */ +export const fromB64URLSafeNoPadding = async (input: string) => { + await sodium.ready; + return sodium.from_base64(input, sodium.base64_variants.URLSAFE_NO_PADDING); +}; + +/** + * Variant of {@link toB64URLSafeNoPadding} that works with {@link strings}. See also + * its sibling method {@link fromB64URLSafeNoPaddingString}. + */ +export const toB64URLSafeNoPaddingString = async (input: string) => { + await sodium.ready; + return toB64URLSafeNoPadding(sodium.from_string(input)); +}; + +/** + * Variant of {@link fromB64URLSafeNoPadding} that works with {@link strings}. See also + * its sibling method {@link toB64URLSafeNoPaddingString}. + */ +export const fromB64URLSafeNoPaddingString = async (input: string) => { + await sodium.ready; + return sodium.to_string(await fromB64URLSafeNoPadding(input)); +}; + +export async function fromUTF8(input: string) { + await sodium.ready; + return sodium.from_string(input); +} + +export async function toUTF8(input: string) { + await sodium.ready; + return sodium.to_string(await fromB64(input)); +} +export async function toHex(input: string) { + await sodium.ready; + return sodium.to_hex(await fromB64(input)); +} + +export async function fromHex(input: string) { + await sodium.ready; + return await toB64(sodium.from_hex(input)); +} + export async function decryptChaChaOneShot( data: Uint8Array, header: Uint8Array, @@ -396,88 +481,3 @@ export async function generateSubKey( ), ); } - -export async function fromB64(input: string) { - await sodium.ready; - return sodium.from_base64(input, sodium.base64_variants.ORIGINAL); -} - -export async function toB64(input: Uint8Array) { - await sodium.ready; - return sodium.to_base64(input, sodium.base64_variants.ORIGINAL); -} - -/** Convert a {@link Uint8Array} to a URL safe Base64 encoded string. */ -export const toB64URLSafe = async (input: Uint8Array) => { - await sodium.ready; - return sodium.to_base64(input, sodium.base64_variants.URLSAFE); -}; - -/** - * Convert a {@link Uint8Array} to a URL safe Base64 encoded string. - * - * This differs from {@link toB64URLSafe} in that it does not append any - * trailing padding character(s) "=" to make the resultant string's length be an - * integer multiple of 4. - * - * - In some contexts, for example when serializing WebAuthn binary for - * transmission over the network, this is the required / recommended - * approach. - * - * - In other cases, for example when trying to pass an arbitrary JSON string - * via a URL parameter, this is also convenient so that we do not have to - * deal with any ambiguity surrounding the "=" which is also the query - * parameter key value separator. - */ -export const toB64URLSafeNoPadding = async (input: Uint8Array) => { - await sodium.ready; - return sodium.to_base64(input, sodium.base64_variants.URLSAFE_NO_PADDING); -}; - -/** - * Convert a Base64 encoded string to a {@link Uint8Array}. - * - * This is the converse of {@link toB64URLSafeNoPadding}, and does not expect - * its input string's length to be a an integer multiple of 4. - */ -export const fromB64URLSafeNoPadding = async (input: string) => { - await sodium.ready; - return sodium.from_base64(input, sodium.base64_variants.URLSAFE_NO_PADDING); -}; - -/** - * Variant of {@link toB64URLSafeNoPadding} that works with {@link strings}. See also - * its sibling method {@link fromB64URLSafeNoPaddingString}. - */ -export const toB64URLSafeNoPaddingString = async (input: string) => { - await sodium.ready; - return toB64URLSafeNoPadding(sodium.from_string(input)); -}; - -/** - * Variant of {@link fromB64URLSafeNoPadding} that works with {@link strings}. See also - * its sibling method {@link toB64URLSafeNoPaddingString}. - */ -export const fromB64URLSafeNoPaddingString = async (input: string) => { - await sodium.ready; - return sodium.to_string(await fromB64URLSafeNoPadding(input)); -}; - -export async function fromUTF8(input: string) { - await sodium.ready; - return sodium.from_string(input); -} - -export async function toUTF8(input: string) { - await sodium.ready; - return sodium.to_string(await fromB64(input)); -} -export async function toHex(input: string) { - await sodium.ready; - return sodium.to_hex(await fromB64(input)); -} - -export async function fromHex(input: string) { - await sodium.ready; - return await toB64(sodium.from_hex(input)); -} From 1496b60895d70a3312af611e6d336d77544617d3 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 26 Jun 2024 19:07:48 +0530 Subject: [PATCH 05/34] Inline --- web/apps/photos/src/services/upload/uploadService.ts | 2 +- web/packages/shared/crypto/constants.ts | 1 - web/packages/shared/crypto/internal/libsodium.ts | 3 ++- 3 files changed, 3 insertions(+), 3 deletions(-) delete mode 100644 web/packages/shared/crypto/constants.ts diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index 66c0dcf8da..b18e0c4781 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -16,7 +16,7 @@ import { basename } from "@/next/file"; import log from "@/next/log"; import { CustomErrorMessage } from "@/next/types/ipc"; import { ensure } from "@/utils/ensure"; -import { ENCRYPTION_CHUNK_SIZE } from "@ente/shared/crypto/constants"; +import { ENCRYPTION_CHUNK_SIZE } from "@ente/shared/crypto/internal/libsodium"; import { DedicatedCryptoWorker } from "@ente/shared/crypto/internal/crypto.worker"; import type { B64EncryptionResult } from "@ente/shared/crypto/types"; import { CustomError, handleUploadError } from "@ente/shared/error"; diff --git a/web/packages/shared/crypto/constants.ts b/web/packages/shared/crypto/constants.ts deleted file mode 100644 index 9226ed8744..0000000000 --- a/web/packages/shared/crypto/constants.ts +++ /dev/null @@ -1 +0,0 @@ -export const ENCRYPTION_CHUNK_SIZE = 4 * 1024 * 1024; diff --git a/web/packages/shared/crypto/internal/libsodium.ts b/web/packages/shared/crypto/internal/libsodium.ts index 8c64bc4b0d..4337a0fb12 100644 --- a/web/packages/shared/crypto/internal/libsodium.ts +++ b/web/packages/shared/crypto/internal/libsodium.ts @@ -9,7 +9,6 @@ import { mergeUint8Arrays } from "@/utils/array"; import { CustomError } from "@ente/shared/error"; import sodium, { type StateAddress } from "libsodium-wrappers"; -import { ENCRYPTION_CHUNK_SIZE } from "../constants"; import type { B64EncryptionResult } from "../types"; export async function fromB64(input: string) { @@ -115,6 +114,8 @@ export async function decryptChaChaOneShot( return pullResult.message; } +export const ENCRYPTION_CHUNK_SIZE = 4 * 1024 * 1024; + export const decryptChaCha = async ( data: Uint8Array, header: Uint8Array, From a9e0aa13ff95fa1d644ee79d18c250bb03fb8c39 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 26 Jun 2024 19:09:35 +0530 Subject: [PATCH 06/34] Inline --- web/apps/photos/src/services/upload/uploadService.ts | 4 ++-- web/packages/accounts/api/user.ts | 2 +- web/packages/accounts/pages/credentials.tsx | 2 +- web/packages/accounts/pages/two-factor/recover.tsx | 2 +- web/packages/shared/crypto/internal/libsodium.ts | 8 +++++++- web/packages/shared/crypto/types.ts | 5 ----- web/packages/shared/user/index.ts | 2 +- 7 files changed, 13 insertions(+), 12 deletions(-) delete mode 100644 web/packages/shared/crypto/types.ts diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index b18e0c4781..c50407eb94 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -16,9 +16,9 @@ import { basename } from "@/next/file"; import log from "@/next/log"; import { CustomErrorMessage } from "@/next/types/ipc"; import { ensure } from "@/utils/ensure"; -import { ENCRYPTION_CHUNK_SIZE } from "@ente/shared/crypto/internal/libsodium"; import { DedicatedCryptoWorker } from "@ente/shared/crypto/internal/crypto.worker"; -import type { B64EncryptionResult } from "@ente/shared/crypto/types"; +import type { B64EncryptionResult } from "@ente/shared/crypto/internal/libsodium"; +import { ENCRYPTION_CHUNK_SIZE } from "@ente/shared/crypto/internal/libsodium"; import { CustomError, handleUploadError } from "@ente/shared/error"; import type { Remote } from "comlink"; import { diff --git a/web/packages/accounts/api/user.ts b/web/packages/accounts/api/user.ts index 131ae73ef7..9269fe2e17 100644 --- a/web/packages/accounts/api/user.ts +++ b/web/packages/accounts/api/user.ts @@ -7,7 +7,7 @@ import type { TwoFactorVerificationResponse, UserVerificationResponse, } from "@ente/accounts/types/user"; -import type { B64EncryptionResult } from "@ente/shared/crypto/types"; +import type { B64EncryptionResult } from "@ente/shared/crypto/internal/libsodium"; import { ApiError, CustomError } from "@ente/shared/error"; import HTTPService from "@ente/shared/network/HTTPService"; import { getToken } from "@ente/shared/storage/localStorage/helpers"; diff --git a/web/packages/accounts/pages/credentials.tsx b/web/packages/accounts/pages/credentials.tsx index 60a2611e7e..39e1c5135a 100644 --- a/web/packages/accounts/pages/credentials.tsx +++ b/web/packages/accounts/pages/credentials.tsx @@ -20,7 +20,7 @@ import { generateLoginSubKey, saveKeyInSessionStore, } from "@ente/shared/crypto/helpers"; -import type { B64EncryptionResult } from "@ente/shared/crypto/types"; +import type { B64EncryptionResult } from "@ente/shared/crypto/internal/libsodium"; import { CustomError } from "@ente/shared/error"; import InMemoryStore, { MS_KEYS } from "@ente/shared/storage/InMemoryStore"; import { diff --git a/web/packages/accounts/pages/two-factor/recover.tsx b/web/packages/accounts/pages/two-factor/recover.tsx index 22608c607f..e52f90c633 100644 --- a/web/packages/accounts/pages/two-factor/recover.tsx +++ b/web/packages/accounts/pages/two-factor/recover.tsx @@ -18,7 +18,7 @@ import SingleInputForm, { } from "@ente/shared/components/SingleInputForm"; import { SUPPORT_EMAIL } from "@ente/shared/constants/urls"; import ComlinkCryptoWorker from "@ente/shared/crypto"; -import type { B64EncryptionResult } from "@ente/shared/crypto/types"; +import type { B64EncryptionResult } from "@ente/shared/crypto/internal/libsodium"; import { ApiError } from "@ente/shared/error"; import { LS_KEYS, getData, setData } from "@ente/shared/storage/localStorage"; import { Link } from "@mui/material"; diff --git a/web/packages/shared/crypto/internal/libsodium.ts b/web/packages/shared/crypto/internal/libsodium.ts index 4337a0fb12..aa02772e52 100644 --- a/web/packages/shared/crypto/internal/libsodium.ts +++ b/web/packages/shared/crypto/internal/libsodium.ts @@ -9,7 +9,6 @@ import { mergeUint8Arrays } from "@/utils/array"; import { CustomError } from "@ente/shared/error"; import sodium, { type StateAddress } from "libsodium-wrappers"; -import type { B64EncryptionResult } from "../types"; export async function fromB64(input: string) { await sodium.ready; @@ -275,6 +274,13 @@ export async function encryptFileChunk( return pushResult; } + +export interface B64EncryptionResult { + encryptedData: string; + key: string; + nonce: string; +} + export async function encryptToB64(data: string, key: string) { await sodium.ready; const encrypted = await encrypt(await fromB64(data), await fromB64(key)); diff --git a/web/packages/shared/crypto/types.ts b/web/packages/shared/crypto/types.ts deleted file mode 100644 index e591820f08..0000000000 --- a/web/packages/shared/crypto/types.ts +++ /dev/null @@ -1,5 +0,0 @@ -export interface B64EncryptionResult { - encryptedData: string; - key: string; - nonce: string; -} diff --git a/web/packages/shared/user/index.ts b/web/packages/shared/user/index.ts index ba80411f6a..f66a62b4f6 100644 --- a/web/packages/shared/user/index.ts +++ b/web/packages/shared/user/index.ts @@ -1,5 +1,5 @@ import ComlinkCryptoWorker from "@ente/shared/crypto"; -import type { B64EncryptionResult } from "@ente/shared/crypto/types"; +import type { B64EncryptionResult } from "@ente/shared/crypto/internal/libsodium"; import { CustomError } from "@ente/shared/error"; import { getKey, SESSION_KEYS } from "@ente/shared/storage/sessionStorage"; From d6f30546b93c4872526b8c6f02f4ee846c81e291 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 26 Jun 2024 19:11:58 +0530 Subject: [PATCH 07/34] Rearrange --- .../shared/crypto/internal/libsodium.ts | 170 +++++++++--------- 1 file changed, 85 insertions(+), 85 deletions(-) diff --git a/web/packages/shared/crypto/internal/libsodium.ts b/web/packages/shared/crypto/internal/libsodium.ts index aa02772e52..704914b17d 100644 --- a/web/packages/shared/crypto/internal/libsodium.ts +++ b/web/packages/shared/crypto/internal/libsodium.ts @@ -95,91 +95,6 @@ export async function fromHex(input: string) { return await toB64(sodium.from_hex(input)); } -export async function decryptChaChaOneShot( - data: Uint8Array, - header: Uint8Array, - key: string, -) { - await sodium.ready; - const pullState = sodium.crypto_secretstream_xchacha20poly1305_init_pull( - header, - await fromB64(key), - ); - const pullResult = sodium.crypto_secretstream_xchacha20poly1305_pull( - pullState, - data, - null, - ); - return pullResult.message; -} - -export const ENCRYPTION_CHUNK_SIZE = 4 * 1024 * 1024; - -export const decryptChaCha = async ( - data: Uint8Array, - header: Uint8Array, - key: string, -) => { - await sodium.ready; - const pullState = sodium.crypto_secretstream_xchacha20poly1305_init_pull( - header, - await fromB64(key), - ); - const decryptionChunkSize = - ENCRYPTION_CHUNK_SIZE + - sodium.crypto_secretstream_xchacha20poly1305_ABYTES; - let bytesRead = 0; - const decryptedChunks = []; - let tag = sodium.crypto_secretstream_xchacha20poly1305_TAG_MESSAGE; - while (tag !== sodium.crypto_secretstream_xchacha20poly1305_TAG_FINAL) { - let chunkSize = decryptionChunkSize; - if (bytesRead + chunkSize > data.length) { - chunkSize = data.length - bytesRead; - } - const buffer = data.slice(bytesRead, bytesRead + chunkSize); - const pullResult = sodium.crypto_secretstream_xchacha20poly1305_pull( - pullState, - buffer, - ); - if (!pullResult.message) { - throw new Error(CustomError.PROCESSING_FAILED); - } - decryptedChunks.push(pullResult.message); - tag = pullResult.tag; - bytesRead += chunkSize; - } - return mergeUint8Arrays(decryptedChunks); -}; - -export async function initChunkDecryption(header: Uint8Array, key: Uint8Array) { - await sodium.ready; - const pullState = sodium.crypto_secretstream_xchacha20poly1305_init_pull( - header, - key, - ); - const decryptionChunkSize = - ENCRYPTION_CHUNK_SIZE + - sodium.crypto_secretstream_xchacha20poly1305_ABYTES; - const tag = sodium.crypto_secretstream_xchacha20poly1305_TAG_MESSAGE; - return { pullState, decryptionChunkSize, tag }; -} - -export async function decryptFileChunk( - data: Uint8Array, - pullState: StateAddress, -) { - await sodium.ready; - const pullResult = sodium.crypto_secretstream_xchacha20poly1305_pull( - pullState, - data, - ); - if (!pullResult.message) { - throw new Error(CustomError.PROCESSING_FAILED); - } - const newTag = pullResult.tag; - return { decryptedData: pullResult.message, newTag }; -} - export async function encryptChaChaOneShot(data: Uint8Array, key: string) { await sodium.ready; @@ -203,6 +118,8 @@ export async function encryptChaChaOneShot(data: Uint8Array, key: string) { }; } +export const ENCRYPTION_CHUNK_SIZE = 4 * 1024 * 1024; + export const encryptChaCha = async (data: Uint8Array) => { await sodium.ready; @@ -275,6 +192,89 @@ export async function encryptFileChunk( return pushResult; } +export async function decryptChaChaOneShot( + data: Uint8Array, + header: Uint8Array, + key: string, +) { + await sodium.ready; + const pullState = sodium.crypto_secretstream_xchacha20poly1305_init_pull( + header, + await fromB64(key), + ); + const pullResult = sodium.crypto_secretstream_xchacha20poly1305_pull( + pullState, + data, + null, + ); + return pullResult.message; +} + +export const decryptChaCha = async ( + data: Uint8Array, + header: Uint8Array, + key: string, +) => { + await sodium.ready; + const pullState = sodium.crypto_secretstream_xchacha20poly1305_init_pull( + header, + await fromB64(key), + ); + const decryptionChunkSize = + ENCRYPTION_CHUNK_SIZE + + sodium.crypto_secretstream_xchacha20poly1305_ABYTES; + let bytesRead = 0; + const decryptedChunks = []; + let tag = sodium.crypto_secretstream_xchacha20poly1305_TAG_MESSAGE; + while (tag !== sodium.crypto_secretstream_xchacha20poly1305_TAG_FINAL) { + let chunkSize = decryptionChunkSize; + if (bytesRead + chunkSize > data.length) { + chunkSize = data.length - bytesRead; + } + const buffer = data.slice(bytesRead, bytesRead + chunkSize); + const pullResult = sodium.crypto_secretstream_xchacha20poly1305_pull( + pullState, + buffer, + ); + if (!pullResult.message) { + throw new Error(CustomError.PROCESSING_FAILED); + } + decryptedChunks.push(pullResult.message); + tag = pullResult.tag; + bytesRead += chunkSize; + } + return mergeUint8Arrays(decryptedChunks); +}; + +export async function initChunkDecryption(header: Uint8Array, key: Uint8Array) { + await sodium.ready; + const pullState = sodium.crypto_secretstream_xchacha20poly1305_init_pull( + header, + key, + ); + const decryptionChunkSize = + ENCRYPTION_CHUNK_SIZE + + sodium.crypto_secretstream_xchacha20poly1305_ABYTES; + const tag = sodium.crypto_secretstream_xchacha20poly1305_TAG_MESSAGE; + return { pullState, decryptionChunkSize, tag }; +} + +export async function decryptFileChunk( + data: Uint8Array, + pullState: StateAddress, +) { + await sodium.ready; + const pullResult = sodium.crypto_secretstream_xchacha20poly1305_pull( + pullState, + data, + ); + if (!pullResult.message) { + throw new Error(CustomError.PROCESSING_FAILED); + } + const newTag = pullResult.tag; + return { decryptedData: pullResult.message, newTag }; +} + export interface B64EncryptionResult { encryptedData: string; key: string; From 90cffef7bedd3b9633f0b18cd0d0f3e3dea85784 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 26 Jun 2024 19:15:09 +0530 Subject: [PATCH 08/34] Doc --- web/packages/shared/crypto/internal/libsodium.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/web/packages/shared/crypto/internal/libsodium.ts b/web/packages/shared/crypto/internal/libsodium.ts index 704914b17d..847b81424e 100644 --- a/web/packages/shared/crypto/internal/libsodium.ts +++ b/web/packages/shared/crypto/internal/libsodium.ts @@ -15,12 +15,17 @@ export async function fromB64(input: string) { return sodium.from_base64(input, sodium.base64_variants.ORIGINAL); } -export async function toB64(input: Uint8Array) { +/** + * Convert a {@link Uint8Array} to a Base64 encoded string. + * + * See also {@link toB64URLSafe} and {@link toB64URLSafeNoPadding}. + */ +export const toB64 = async (input: Uint8Array) => { await sodium.ready; return sodium.to_base64(input, sodium.base64_variants.ORIGINAL); -} +}; -/** Convert a {@link Uint8Array} to a URL safe Base64 encoded string. */ +/** Convert a {@link Uint8Array} to a **URL safe** Base64 encoded string. */ export const toB64URLSafe = async (input: Uint8Array) => { await sodium.ready; return sodium.to_base64(input, sodium.base64_variants.URLSAFE); From 660f6c645b1bcd3b02585b74e34f9951669e13c9 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 26 Jun 2024 19:15:45 +0530 Subject: [PATCH 09/34] Rearrange --- web/packages/shared/crypto/internal/libsodium.ts | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/web/packages/shared/crypto/internal/libsodium.ts b/web/packages/shared/crypto/internal/libsodium.ts index 847b81424e..413d21aca1 100644 --- a/web/packages/shared/crypto/internal/libsodium.ts +++ b/web/packages/shared/crypto/internal/libsodium.ts @@ -10,11 +10,6 @@ import { mergeUint8Arrays } from "@/utils/array"; import { CustomError } from "@ente/shared/error"; import sodium, { type StateAddress } from "libsodium-wrappers"; -export async function fromB64(input: string) { - await sodium.ready; - return sodium.from_base64(input, sodium.base64_variants.ORIGINAL); -} - /** * Convert a {@link Uint8Array} to a Base64 encoded string. * @@ -25,7 +20,9 @@ export const toB64 = async (input: Uint8Array) => { return sodium.to_base64(input, sodium.base64_variants.ORIGINAL); }; -/** Convert a {@link Uint8Array} to a **URL safe** Base64 encoded string. */ +/** + * Convert a {@link Uint8Array} to a **URL safe** Base64 encoded string. + */ export const toB64URLSafe = async (input: Uint8Array) => { await sodium.ready; return sodium.to_base64(input, sodium.base64_variants.URLSAFE); @@ -52,6 +49,11 @@ export const toB64URLSafeNoPadding = async (input: Uint8Array) => { return sodium.to_base64(input, sodium.base64_variants.URLSAFE_NO_PADDING); }; +export async function fromB64(input: string) { + await sodium.ready; + return sodium.from_base64(input, sodium.base64_variants.ORIGINAL); +} + /** * Convert a Base64 encoded string to a {@link Uint8Array}. * From 27c6474f06eaac338ddcef4f13e3850eddc91def Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 26 Jun 2024 19:21:22 +0530 Subject: [PATCH 10/34] Doc all --- .../shared/crypto/internal/libsodium.ts | 28 ++++++++++++------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/web/packages/shared/crypto/internal/libsodium.ts b/web/packages/shared/crypto/internal/libsodium.ts index 413d21aca1..857c826d5a 100644 --- a/web/packages/shared/crypto/internal/libsodium.ts +++ b/web/packages/shared/crypto/internal/libsodium.ts @@ -21,7 +21,19 @@ export const toB64 = async (input: Uint8Array) => { }; /** - * Convert a {@link Uint8Array} to a **URL safe** Base64 encoded string. + * Convert a Base64 encoded string to a {@link Uint8Array}. + * + * This is the converse of {@link toBase64}. + */ +export const fromB64 = async (input: string) => { + await sodium.ready; + return sodium.from_base64(input, sodium.base64_variants.ORIGINAL); +}; + +/** + * Convert a {@link Uint8Array} to a URL-safe Base64 encoded string. + * + * See also {@link toB64URLSafe} and {@link toB64URLSafeNoPadding}. */ export const toB64URLSafe = async (input: Uint8Array) => { await sodium.ready; @@ -29,7 +41,7 @@ export const toB64URLSafe = async (input: Uint8Array) => { }; /** - * Convert a {@link Uint8Array} to a URL safe Base64 encoded string. + * Convert a {@link Uint8Array} to a unpadded URL-safe Base64 encoded string. * * This differs from {@link toB64URLSafe} in that it does not append any * trailing padding character(s) "=" to make the resultant string's length be an @@ -49,13 +61,8 @@ export const toB64URLSafeNoPadding = async (input: Uint8Array) => { return sodium.to_base64(input, sodium.base64_variants.URLSAFE_NO_PADDING); }; -export async function fromB64(input: string) { - await sodium.ready; - return sodium.from_base64(input, sodium.base64_variants.ORIGINAL); -} - /** - * Convert a Base64 encoded string to a {@link Uint8Array}. + * Convert a unpadded URL-safe Base64 encoded string to a {@link Uint8Array}. * * This is the converse of {@link toB64URLSafeNoPadding}, and does not expect * its input string's length to be a an integer multiple of 4. @@ -66,8 +73,8 @@ export const fromB64URLSafeNoPadding = async (input: string) => { }; /** - * Variant of {@link toB64URLSafeNoPadding} that works with {@link strings}. See also - * its sibling method {@link fromB64URLSafeNoPaddingString}. + * Variant of {@link toB64URLSafeNoPadding} that works with {@link string} + * inputs. See also its sibling method {@link fromB64URLSafeNoPaddingString}. */ export const toB64URLSafeNoPaddingString = async (input: string) => { await sodium.ready; @@ -92,6 +99,7 @@ export async function toUTF8(input: string) { await sodium.ready; return sodium.to_string(await fromB64(input)); } + export async function toHex(input: string) { await sodium.ready; return sodium.to_hex(await fromB64(input)); From 45a103f66a53f8e10569e06e6c1fd8f75a1fac9c Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 26 Jun 2024 19:37:19 +0530 Subject: [PATCH 11/34] New layer --- web/packages/new/common/crypto/ente.ts | 37 ++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 web/packages/new/common/crypto/ente.ts diff --git a/web/packages/new/common/crypto/ente.ts b/web/packages/new/common/crypto/ente.ts new file mode 100644 index 0000000000..292029dd13 --- /dev/null +++ b/web/packages/new/common/crypto/ente.ts @@ -0,0 +1,37 @@ +/** + * @file Higher level functions that use the ontology of Ente's types + * + * These are thin wrappers over the (thin-) wrappers in internal/libsodium.ts. + * The main difference is that they don't name things in terms of the crypto + * algorithms, but rather by the specific Ente specific tasks we are trying to + * do. + */ +import * as libsodium from "@ente/shared/crypto/internal/libsodium"; + +/** + * Decrypt the metadata associated with a file using the file's key. + * + * @param encryptedMetadataB64 The Base64 encoded string containing the + * encrypted data. + * + * @param headerB64 The Base64 encoded string containing the decryption header + * produced during encryption. + * + * @param keyB64 The Base64 encoded string containing the encryption key (which + * is the file's key). + * + * @returns The decrypted utf-8 string. + */ +export const decryptFileMetadata = async ( + encryptedMetadataB64: string, + decryptionHeaderB64: string, + keyB64: string, +) => { + const metadataBytes = await libsodium.decryptChaChaOneShot( + await libsodium.fromB64(encryptedMetadataB64), + await libsodium.fromB64(decryptionHeaderB64), + keyB64, + ); + const textDecoder = new TextDecoder(); + return textDecoder.decode(metadataBytes); +}; From 593ece76819960191e96253ec5156a28e4aeaeef Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 26 Jun 2024 19:52:57 +0530 Subject: [PATCH 12/34] Compile --- web/packages/new/common/crypto/ente.ts | 6 ++--- web/packages/new/photos/services/embedding.ts | 26 ++++++++++++------- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/web/packages/new/common/crypto/ente.ts b/web/packages/new/common/crypto/ente.ts index 292029dd13..4995942a87 100644 --- a/web/packages/new/common/crypto/ente.ts +++ b/web/packages/new/common/crypto/ente.ts @@ -9,7 +9,7 @@ import * as libsodium from "@ente/shared/crypto/internal/libsodium"; /** - * Decrypt the metadata associated with a file using the file's key. + * Decrypt arbitrary metadata associated with a file using the its's key. * * @param encryptedMetadataB64 The Base64 encoded string containing the * encrypted data. @@ -17,8 +17,8 @@ import * as libsodium from "@ente/shared/crypto/internal/libsodium"; * @param headerB64 The Base64 encoded string containing the decryption header * produced during encryption. * - * @param keyB64 The Base64 encoded string containing the encryption key (which - * is the file's key). + * @param keyB64 The Base64 encoded string containing the encryption key + * (this'll generally be the file's key). * * @returns The decrypted utf-8 string. */ diff --git a/web/packages/new/photos/services/embedding.ts b/web/packages/new/photos/services/embedding.ts index 5b422eaff1..977d396e20 100644 --- a/web/packages/new/photos/services/embedding.ts +++ b/web/packages/new/photos/services/embedding.ts @@ -1,8 +1,9 @@ import { authenticatedRequestHeaders } from "@/next/http"; import { apiURL } from "@/next/origins"; +import log from "@/next/log"; import { nullToUndefined } from "@/utils/transform"; -import ComlinkCryptoWorker from "@ente/shared/crypto"; import { z } from "zod"; +import { decryptFileMetadata } from "../../common/crypto/ente"; import { getAllLocalFiles } from "./files"; /** @@ -83,12 +84,11 @@ type RemoteEmbedding = z.infer; */ export const syncRemoteFaceEmbeddings = async () => { let sinceTime = faceEmbeddingSyncTime(); - const cryptoWorker = await ComlinkCryptoWorker.getInstance(); const localFiles = await getAllLocalFiles(); const localFilesByID = new Map(localFiles.map((f) => [f.id, f])); const decryptEmbedding = async (remoteEmbedding: RemoteEmbedding) => { - const file = localFilesByID.get(remoteEmbedding.fileID) + const file = localFilesByID.get(remoteEmbedding.fileID); // [Note: Ignoring embeddings for unknown files] // // We need the file to decrypt the embedding. This is easily ensured by @@ -112,8 +112,18 @@ export const syncRemoteFaceEmbeddings = async () => { // be a bit of duplicate work, but that's fine as long as there // isn't a systematic scenario where this happens. if (!file) return undefined; - - } + try { + const decryptedString = await decryptFileMetadata( + remoteEmbedding.encryptedEmbedding, + remoteEmbedding.decryptionHeader, + file.key, + ); + return decryptedString; + } catch (e) { + log.warn("Ignoring unparseable embedding", e); + return undefined; + } + }; // TODO: eslint has fixed this spurious warning, but we're not on the latest // version yet, so add a disable. @@ -126,9 +136,7 @@ export const syncRemoteFaceEmbeddings = async () => { sinceTime, ); if (remoteEmbeddings.length == 0) break; - // const _embeddings = Promise.all( - // remoteEmbeddings.map(decryptEmbedding), - // ); + void (await Promise.all(remoteEmbeddings.map(decryptEmbedding))); sinceTime = remoteEmbeddings.reduce( (max, { updatedAt }) => Math.max(max, updatedAt), sinceTime, @@ -137,7 +145,7 @@ export const syncRemoteFaceEmbeddings = async () => { } }; -const decryptFaceEmbedding = async (remoteEmbedding: RemoteEmbedding) => { +// const decryptFaceEmbedding = async (remoteEmbedding: RemoteEmbedding) => { // const fileKey = fileIdToKeyMap.get(embedding.fileID); // if (!fileKey) { // throw Error(CustomError.FILE_NOT_FOUND); From 61d35159fa2d3c1592b663f4c684f37c6429030a Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 26 Jun 2024 20:23:12 +0530 Subject: [PATCH 13/34] Dup --- web/packages/new/photos/services/embedding.ts | 71 ++++++++++++++++++- 1 file changed, 68 insertions(+), 3 deletions(-) diff --git a/web/packages/new/photos/services/embedding.ts b/web/packages/new/photos/services/embedding.ts index 977d396e20..ef01b81b35 100644 --- a/web/packages/new/photos/services/embedding.ts +++ b/web/packages/new/photos/services/embedding.ts @@ -175,9 +175,74 @@ const faceEmbeddingSyncTime = () => const saveFaceEmbeddingSyncTime = (t: number) => localStorage.setItem("faceEmbeddingSyncTime", `${t}`); -// const getFaceEmbeddings = async () => { - -// } +/** + * Zod schemas for the {@link FaceIndex} types. + * + * [Note: Duplicated between Zod schemas and TS type] + * + * Usually we define a Zod schema, and then infer the corresponding TypeScript + * type for it using `z.infer`. This works great except now the docstrings don't + * show up: The doc strings get added to the Zod schema, but usually the code + * using the parsed data will reference the TypeScript type, and the docstrings + * added to the fields in the Zod schema won't show up. + * + * We usually live with this infelicity, since the alternative is code + * duplication: Define the TypeScript type (putting the docstrings therein) + * _and_ also a corresponding Zod schema. The duplication happens because it is + * not possible to go the other way (TS type => Zod schema). + * + * However, in some cases having when the TypeScript type under consideration is + * used pervasely in our code, having a standalone TypeScript type with attached + * docstrings, is worth the code duplication. + * + * Note that this'll just be syntactic duplication - if the two definitions get + * out of sync in the shape of the types they represent, the TypeScript compiler + * will flag it for us. + */ +const FaceIndex = z + .object({ + fileID: z.number(), + width: z.number(), + height: z.number(), + faceEmbedding: z + .object({ + version: z.number(), + client: z.string(), + faces: z.array( + z + .object({ + faceID: z.string(), + detection: z + .object({ + box: z + .object({ + x: z.number(), + y: z.number(), + width: z.number(), + height: z.number(), + }) + .passthrough(), + landmarks: z.array( + z + .object({ + x: z.number(), + y: z.number(), + }) + .passthrough(), + ), + }) + .passthrough(), + score: z.number(), + blur: z.number(), + embedding: z.array(z.number()), + }) + .passthrough(), + ), + }) + .passthrough(), + }) + // Retain fields we might not (currently) understand. + .passthrough(); /** * The maximum number of items to fetch in a single GET /embeddings/diff From 67d9e650ba83aba75acc6f04c0174c18b87261ef Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 26 Jun 2024 20:59:20 +0530 Subject: [PATCH 14/34] Include files from trash --- web/apps/photos/src/pages/gallery/index.tsx | 9 ++-- .../photos/src/pages/shared-albums/index.tsx | 8 +--- .../photos/src/services/embeddingService.ts | 6 ++- web/apps/photos/src/services/export/index.ts | 7 +--- .../photos/src/services/export/migration.ts | 7 +--- web/apps/photos/src/services/fileService.ts | 8 +--- .../src/services/publicCollectionService.ts | 3 +- web/apps/photos/src/services/trashService.ts | 33 ++++----------- web/apps/photos/src/types/trash/index.ts | 16 -------- web/apps/photos/src/utils/file/index.ts | 40 +----------------- web/packages/new/photos/services/embedding.ts | 8 +++- web/packages/new/photos/services/files.ts | 41 ++++++++++++++++++- web/packages/new/photos/types/file.ts | 15 +++++++ web/packages/new/photos/utils/file.ts | 30 ++++++++++++++ 14 files changed, 120 insertions(+), 111 deletions(-) delete mode 100644 web/apps/photos/src/types/trash/index.ts create mode 100644 web/packages/new/photos/utils/file.ts diff --git a/web/apps/photos/src/pages/gallery/index.tsx b/web/apps/photos/src/pages/gallery/index.tsx index 360100c01c..a5c180c3aa 100644 --- a/web/apps/photos/src/pages/gallery/index.tsx +++ b/web/apps/photos/src/pages/gallery/index.tsx @@ -1,8 +1,12 @@ import { WhatsNew } from "@/new/photos/components/WhatsNew"; import { shouldShowWhatsNew } from "@/new/photos/services/changelog"; import { fetchAndSaveFeatureFlagsIfNeeded } from "@/new/photos/services/feature-flags"; -import { getLocalFiles } from "@/new/photos/services/files"; +import { + getLocalFiles, + getLocalTrashedFiles, +} from "@/new/photos/services/files"; import { EnteFile } from "@/new/photos/types/file"; +import { mergeMetadata } from "@/new/photos/utils/file"; import log from "@/next/log"; import { CenteredFlex } from "@ente/shared/components/Container"; import EnteSpinner from "@ente/shared/components/EnteSpinner"; @@ -94,7 +98,7 @@ import { syncCLIPEmbeddings } from "services/embeddingService"; import { syncEntities } from "services/entityService"; import { syncFiles } from "services/fileService"; import locationSearchService from "services/locationSearchService"; -import { getLocalTrashedFiles, syncTrash } from "services/trashService"; +import { syncTrash } from "services/trashService"; import uploadManager from "services/upload/uploadManager"; import { isTokenValid, syncMapEnabled } from "services/userService"; import { Collection, CollectionSummaries } from "types/collection"; @@ -125,7 +129,6 @@ import { getSelectedFiles, getUniqueFiles, handleFileOps, - mergeMetadata, sortFiles, } from "utils/file"; import { isArchivedFile } from "utils/magicMetadata"; diff --git a/web/apps/photos/src/pages/shared-albums/index.tsx b/web/apps/photos/src/pages/shared-albums/index.tsx index 2976ed1e88..4e479457b1 100644 --- a/web/apps/photos/src/pages/shared-albums/index.tsx +++ b/web/apps/photos/src/pages/shared-albums/index.tsx @@ -1,4 +1,5 @@ import { EnteFile } from "@/new/photos/types/file"; +import { mergeMetadata } from "@/new/photos/utils/file"; import log from "@/next/log"; import { CenteredFlex, @@ -65,12 +66,7 @@ import { SetFilesDownloadProgressAttributesCreator, } from "types/gallery"; import { downloadCollectionFiles, isHiddenCollection } from "utils/collection"; -import { - downloadSelectedFiles, - getSelectedFiles, - mergeMetadata, - sortFiles, -} from "utils/file"; +import { downloadSelectedFiles, getSelectedFiles, sortFiles } from "utils/file"; import { PublicCollectionGalleryContext } from "utils/publicCollectionGallery"; export default function PublicCollectionGallery() { diff --git a/web/apps/photos/src/services/embeddingService.ts b/web/apps/photos/src/services/embeddingService.ts index 4aa6c39878..859557cbc9 100644 --- a/web/apps/photos/src/services/embeddingService.ts +++ b/web/apps/photos/src/services/embeddingService.ts @@ -1,5 +1,8 @@ import type { EmbeddingModel } from "@/new/photos/services/embedding"; -import { getAllLocalFiles } from "@/new/photos/services/files"; +import { + getAllLocalFiles, + getLocalTrashedFiles, +} from "@/new/photos/services/files"; import { EnteFile } from "@/new/photos/types/file"; import { inWorker } from "@/next/env"; import log from "@/next/log"; @@ -18,7 +21,6 @@ import type { } from "types/embedding"; import { getLocalCollections } from "./collectionService"; import type { FaceIndex } from "./face/types"; -import { getLocalTrashedFiles } from "./trashService"; type FileML = FaceIndex & { updatedAt: number; diff --git a/web/apps/photos/src/services/export/index.ts b/web/apps/photos/src/services/export/index.ts index 4afe2e7bcc..df7d23eddf 100644 --- a/web/apps/photos/src/services/export/index.ts +++ b/web/apps/photos/src/services/export/index.ts @@ -3,6 +3,7 @@ import { decodeLivePhoto } from "@/media/live-photo"; import type { Metadata } from "@/media/types/file"; import { getAllLocalFiles } from "@/new/photos/services/files"; import { EnteFile } from "@/new/photos/types/file"; +import { mergeMetadata } from "@/new/photos/utils/file"; import { ensureElectron } from "@/next/electron"; import log from "@/next/log"; import { wait } from "@/utils/promise"; @@ -29,11 +30,7 @@ import { getCollectionUserFacingName, getNonEmptyPersonalCollections, } from "utils/collection"; -import { - getPersonalFiles, - getUpdatedEXIFFileForDownload, - mergeMetadata, -} from "utils/file"; +import { getPersonalFiles, getUpdatedEXIFFileForDownload } from "utils/file"; import { safeDirectoryName, safeFileName } from "utils/native-fs"; import { writeStream } from "utils/native-stream"; import { getAllLocalCollections } from "../collectionService"; diff --git a/web/apps/photos/src/services/export/migration.ts b/web/apps/photos/src/services/export/migration.ts index f452a52642..75ab0e2f11 100644 --- a/web/apps/photos/src/services/export/migration.ts +++ b/web/apps/photos/src/services/export/migration.ts @@ -2,6 +2,7 @@ import { FILE_TYPE } from "@/media/file-type"; import { decodeLivePhoto } from "@/media/live-photo"; import { getAllLocalFiles } from "@/new/photos/services/files"; import { EnteFile } from "@/new/photos/types/file"; +import { mergeMetadata } from "@/new/photos/utils/file"; import { ensureElectron } from "@/next/electron"; import { nameAndExtension } from "@/next/file"; import log from "@/next/log"; @@ -22,11 +23,7 @@ import { FileExportNames, } from "types/export"; import { getNonEmptyPersonalCollections } from "utils/collection"; -import { - getIDBasedSortedFiles, - getPersonalFiles, - mergeMetadata, -} from "utils/file"; +import { getIDBasedSortedFiles, getPersonalFiles } from "utils/file"; import { safeDirectoryName, safeFileName, diff --git a/web/apps/photos/src/services/fileService.ts b/web/apps/photos/src/services/fileService.ts index 968fdf4df0..14ad8ddfaa 100644 --- a/web/apps/photos/src/services/fileService.ts +++ b/web/apps/photos/src/services/fileService.ts @@ -7,6 +7,7 @@ import { TrashRequest, } from "@/new/photos/types/file"; import { BulkUpdateMagicMetadataRequest } from "@/new/photos/types/magicMetadata"; +import { mergeMetadata } from "@/new/photos/utils/file"; import log from "@/next/log"; import { apiURL } from "@/next/origins"; import ComlinkCryptoWorker from "@ente/shared/crypto"; @@ -16,12 +17,7 @@ import { REQUEST_BATCH_SIZE } from "constants/api"; import { Collection } from "types/collection"; import { SetFiles } from "types/gallery"; import { batch } from "utils/common"; -import { - decryptFile, - getLatestVersionFiles, - mergeMetadata, - sortFiles, -} from "utils/file"; +import { decryptFile, getLatestVersionFiles, sortFiles } from "utils/file"; import { getCollectionLastSyncTime, setCollectionLastSyncTime, diff --git a/web/apps/photos/src/services/publicCollectionService.ts b/web/apps/photos/src/services/publicCollectionService.ts index 0528094d3f..e680f6c490 100644 --- a/web/apps/photos/src/services/publicCollectionService.ts +++ b/web/apps/photos/src/services/publicCollectionService.ts @@ -1,4 +1,5 @@ import { EncryptedEnteFile, EnteFile } from "@/new/photos/types/file"; +import { mergeMetadata } from "@/new/photos/utils/file"; import log from "@/next/log"; import { apiURL } from "@/next/origins"; import ComlinkCryptoWorker from "@ente/shared/crypto"; @@ -7,7 +8,7 @@ import HTTPService from "@ente/shared/network/HTTPService"; import localForage from "@ente/shared/storage/localForage"; import { Collection, CollectionPublicMagicMetadata } from "types/collection"; import { LocalSavedPublicCollectionFiles } from "types/publicCollection"; -import { decryptFile, mergeMetadata, sortFiles } from "utils/file"; +import { decryptFile, sortFiles } from "utils/file"; const PUBLIC_COLLECTION_FILES_TABLE = "public-collection-files"; const PUBLIC_COLLECTIONS_TABLE = "public-collections"; diff --git a/web/apps/photos/src/services/trashService.ts b/web/apps/photos/src/services/trashService.ts index b4920ed1ea..b3762be3bf 100644 --- a/web/apps/photos/src/services/trashService.ts +++ b/web/apps/photos/src/services/trashService.ts @@ -1,4 +1,9 @@ -import { EnteFile } from "@/new/photos/types/file"; +import { + getLocalTrash, + getTrashedFiles, + TRASH, +} from "@/new/photos/services/files"; +import { EncryptedTrashItem, Trash } from "@/new/photos/types/file"; import log from "@/next/log"; import { apiURL } from "@/next/origins"; import HTTPService from "@ente/shared/network/HTTPService"; @@ -6,23 +11,12 @@ import localForage from "@ente/shared/storage/localForage"; import { getToken } from "@ente/shared/storage/localStorage/helpers"; import { Collection } from "types/collection"; import { SetFiles } from "types/gallery"; -import { EncryptedTrashItem, Trash } from "types/trash"; -import { decryptFile, mergeMetadata, sortTrashFiles } from "utils/file"; +import { decryptFile } from "utils/file"; import { getCollection } from "./collectionService"; -const TRASH = "file-trash"; const TRASH_TIME = "trash-time"; const DELETED_COLLECTION = "deleted-collection"; -async function getLocalTrash() { - const trash = (await localForage.getItem(TRASH)) || []; - return trash; -} - -export async function getLocalTrashedFiles() { - return getTrashedFiles(await getLocalTrash()); -} - export async function getLocalDeletedCollections() { const trashedCollections: Array = (await localForage.getItem(DELETED_COLLECTION)) || []; @@ -136,19 +130,6 @@ export const updateTrash = async ( return currentTrash; }; -export function getTrashedFiles(trash: Trash): EnteFile[] { - return sortTrashFiles( - mergeMetadata( - trash.map((trashedFile) => ({ - ...trashedFile.file, - updationTime: trashedFile.updatedAt, - deleteBy: trashedFile.deleteBy, - isTrashed: true, - })), - ), - ); -} - export const emptyTrash = async () => { try { const token = getToken(); diff --git a/web/apps/photos/src/types/trash/index.ts b/web/apps/photos/src/types/trash/index.ts deleted file mode 100644 index df786029a1..0000000000 --- a/web/apps/photos/src/types/trash/index.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { EncryptedEnteFile, EnteFile } from "@/new/photos/types/file"; - -export interface TrashItem extends Omit { - file: EnteFile; -} - -export interface EncryptedTrashItem { - file: EncryptedEnteFile; - isDeleted: boolean; - isRestored: boolean; - deleteBy: number; - createdAt: number; - updatedAt: number; -} - -export type Trash = TrashItem[]; diff --git a/web/apps/photos/src/utils/file/index.ts b/web/apps/photos/src/utils/file/index.ts index a6793a3386..f9bb04b94c 100644 --- a/web/apps/photos/src/utils/file/index.ts +++ b/web/apps/photos/src/utils/file/index.ts @@ -12,6 +12,7 @@ import { FileWithUpdatedMagicMetadata, } from "@/new/photos/types/file"; import { VISIBILITY_STATE } from "@/new/photos/types/magicMetadata"; +import { mergeMetadata } from "@/new/photos/utils/file"; import { lowercaseExtension } from "@/next/file"; import log from "@/next/log"; import { CustomErrorMessage, type Electron } from "@/next/types/ipc"; @@ -197,20 +198,6 @@ export function sortFiles(files: EnteFile[], sortAsc = false) { }); } -export function sortTrashFiles(files: EnteFile[]) { - return files.sort((a, b) => { - if (a.deleteBy === b.deleteBy) { - if (a.metadata.creationTime === b.metadata.creationTime) { - return ( - b.metadata.modificationTime - a.metadata.modificationTime - ); - } - return b.metadata.creationTime - a.metadata.creationTime; - } - return a.deleteBy - b.deleteBy; - }); -} - export async function decryptFile( file: EncryptedEnteFile, collectionKey: string, @@ -432,31 +419,6 @@ export function isSharedFile(user: User, file: EnteFile) { return file.ownerID !== user.id; } -/** - * [Note: File name for local EnteFile objects] - * - * The title property in a file's metadata is the original file's name. The - * metadata of a file cannot be edited. So if later on the file's name is - * changed, then the edit is stored in the `editedName` property of the public - * metadata of the file. - * - * This function merges these edits onto the file object that we use locally. - * Effectively, post this step, the file's metadata.title can be used in lieu of - * its filename. - */ -export function mergeMetadata(files: EnteFile[]): EnteFile[] { - return files.map((file) => { - if (file.pubMagicMetadata?.data.editedTime) { - file.metadata.creationTime = file.pubMagicMetadata.data.editedTime; - } - if (file.pubMagicMetadata?.data.editedName) { - file.metadata.title = file.pubMagicMetadata.data.editedName; - } - - return file; - }); -} - export function updateExistingFilePubMetadata( existingFile: EnteFile, updatedFile: EnteFile, diff --git a/web/packages/new/photos/services/embedding.ts b/web/packages/new/photos/services/embedding.ts index ef01b81b35..a8c37ade97 100644 --- a/web/packages/new/photos/services/embedding.ts +++ b/web/packages/new/photos/services/embedding.ts @@ -1,3 +1,4 @@ +import { getLocalTrashedFiles } from "@/new/photos/services/files"; import { authenticatedRequestHeaders } from "@/next/http"; import { apiURL } from "@/next/origins"; import log from "@/next/log"; @@ -84,7 +85,11 @@ type RemoteEmbedding = z.infer; */ export const syncRemoteFaceEmbeddings = async () => { let sinceTime = faceEmbeddingSyncTime(); - const localFiles = await getAllLocalFiles(); + // Include files from trash, otherwise they'll get unnecessarily reindexed + // if the user restores them from trash before permanent deletion. + const localFiles = (await getAllLocalFiles()).concat( + await getLocalTrashedFiles(), + ); const localFilesByID = new Map(localFiles.map((f) => [f.id, f])); const decryptEmbedding = async (remoteEmbedding: RemoteEmbedding) => { @@ -199,6 +204,7 @@ const saveFaceEmbeddingSyncTime = (t: number) => * out of sync in the shape of the types they represent, the TypeScript compiler * will flag it for us. */ +// eslint-disable-next-line @typescript-eslint/no-unused-vars const FaceIndex = z .object({ fileID: z.number(), diff --git a/web/packages/new/photos/services/files.ts b/web/packages/new/photos/services/files.ts index d5a17b4e72..ddb24488df 100644 --- a/web/packages/new/photos/services/files.ts +++ b/web/packages/new/photos/services/files.ts @@ -1,7 +1,8 @@ -import { type EnteFile } from "@/new/photos/types/file"; import log from "@/next/log"; import { Events, eventBus } from "@ente/shared/events"; import localForage from "@ente/shared/storage/localForage"; +import { type EnteFile, type Trash } from "../types/file"; +import { mergeMetadata } from "../utils/file"; const FILES_TABLE = "files"; const HIDDEN_FILES_TABLE = "hidden-files"; @@ -36,3 +37,41 @@ export const setLocalFiles = async ( log.error("Failed to save files", e); } }; + +export const TRASH = "file-trash"; + +export async function getLocalTrash() { + const trash = (await localForage.getItem(TRASH)) ?? []; + return trash; +} + +export async function getLocalTrashedFiles() { + return getTrashedFiles(await getLocalTrash()); +} + +export function getTrashedFiles(trash: Trash): EnteFile[] { + return sortTrashFiles( + mergeMetadata( + trash.map((trashedFile) => ({ + ...trashedFile.file, + updationTime: trashedFile.updatedAt, + deleteBy: trashedFile.deleteBy, + isTrashed: true, + })), + ), + ); +} + +const sortTrashFiles = (files: EnteFile[]) => { + return files.sort((a, b) => { + if (a.deleteBy === b.deleteBy) { + if (a.metadata.creationTime === b.metadata.creationTime) { + return ( + b.metadata.modificationTime - a.metadata.modificationTime + ); + } + return b.metadata.creationTime - a.metadata.creationTime; + } + return (a.deleteBy ?? 0) - (b.deleteBy ?? 0); + }); +}; diff --git a/web/packages/new/photos/types/file.ts b/web/packages/new/photos/types/file.ts index 1993ff9e8b..e76e527393 100644 --- a/web/packages/new/photos/types/file.ts +++ b/web/packages/new/photos/types/file.ts @@ -126,3 +126,18 @@ export interface FilePublicMagicMetadataProps { export type FilePublicMagicMetadata = MagicMetadataCore; + +export interface TrashItem extends Omit { + file: EnteFile; +} + +export interface EncryptedTrashItem { + file: EncryptedEnteFile; + isDeleted: boolean; + isRestored: boolean; + deleteBy: number; + createdAt: number; + updatedAt: number; +} + +export type Trash = TrashItem[]; diff --git a/web/packages/new/photos/utils/file.ts b/web/packages/new/photos/utils/file.ts new file mode 100644 index 0000000000..d5d2892254 --- /dev/null +++ b/web/packages/new/photos/utils/file.ts @@ -0,0 +1,30 @@ +import type { EnteFile } from "../types/file"; + +/** + * [Note: File name for local EnteFile objects] + * + * The title property in a file's metadata is the original file's name. The + * metadata of a file cannot be edited. So if later on the file's name is + * changed, then the edit is stored in the `editedName` property of the public + * metadata of the file. + * + * This function merges these edits onto the file object that we use locally. + * Effectively, post this step, the file's metadata.title can be used in lieu of + * its filename. + */ +export function mergeMetadata(files: EnteFile[]): EnteFile[] { + return files.map((file) => { + // TODO: Until the types reflect reality + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + if (file.pubMagicMetadata?.data.editedTime) { + file.metadata.creationTime = file.pubMagicMetadata.data.editedTime; + } + // TODO: Until the types reflect reality + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + if (file.pubMagicMetadata?.data.editedName) { + file.metadata.title = file.pubMagicMetadata.data.editedName; + } + + return file; + }); +} From a119d544af3e94f00517a5c563607f660eb5c3b0 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 27 Jun 2024 09:25:48 +0530 Subject: [PATCH 15/34] prune prep --- web/apps/photos/src/services/face/indexer.ts | 12 ++++++------ web/apps/photos/src/services/face/indexer.worker.ts | 8 ++++---- web/apps/photos/src/services/logout.ts | 2 +- web/packages/new/photos/services/embedding.ts | 6 +++++- .../src => packages/new/photos}/services/face/db.ts | 0 .../new/photos}/services/face/types.ts | 0 6 files changed, 16 insertions(+), 12 deletions(-) rename web/{apps/photos/src => packages/new/photos}/services/face/db.ts (100%) rename web/{apps/photos/src => packages/new/photos}/services/face/types.ts (100%) diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index c6cdc3295a..0f78c20e2e 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -1,3 +1,9 @@ +import { + faceIndex, + indexableFileIDs, + indexedAndIndexableCounts, + syncWithLocalFiles, +} from "@/new/photos/services/face/db"; import { isBetaUser, isInternalUser, @@ -7,12 +13,6 @@ import type { EnteFile } from "@/new/photos/types/file"; import { ComlinkWorker } from "@/next/worker/comlink-worker"; import { ensure } from "@/utils/ensure"; import type { Remote } from "comlink"; -import { - faceIndex, - indexableFileIDs, - indexedAndIndexableCounts, - syncWithLocalFiles, -} from "./db"; import type { FaceIndexerWorker } from "./indexer.worker"; /** diff --git a/web/apps/photos/src/services/face/indexer.worker.ts b/web/apps/photos/src/services/face/indexer.worker.ts index a31db7550d..27608a31c8 100644 --- a/web/apps/photos/src/services/face/indexer.worker.ts +++ b/web/apps/photos/src/services/face/indexer.worker.ts @@ -1,11 +1,11 @@ -import type { EnteFile } from "@/new/photos/types/file"; -import log from "@/next/log"; -import { fileLogID } from "utils/file"; import { closeFaceDBConnectionsIfNeeded, markIndexingFailed, saveFaceIndex, -} from "./db"; +} from "@/new/photos/services/face/db"; +import type { EnteFile } from "@/new/photos/types/file"; +import log from "@/next/log"; +import { fileLogID } from "utils/file"; import { indexFaces } from "./f-index"; import { putFaceIndex } from "./remote"; import type { FaceIndex } from "./types"; diff --git a/web/apps/photos/src/services/logout.ts b/web/apps/photos/src/services/logout.ts index 6d335e83df..1a6c5547e8 100644 --- a/web/apps/photos/src/services/logout.ts +++ b/web/apps/photos/src/services/logout.ts @@ -1,10 +1,10 @@ +import { clearFaceData } from "@/new/photos/services/face/db"; import { clearFeatureFlagSessionState } from "@/new/photos/services/feature-flags"; import log from "@/next/log"; import { accountLogout } from "@ente/accounts/services/logout"; import { clipService } from "services/clip-service"; import DownloadManager from "./download"; import exportService from "./export"; -import { clearFaceData } from "./face/db"; import mlWorkManager from "./face/mlWorkManager"; /** diff --git a/web/packages/new/photos/services/embedding.ts b/web/packages/new/photos/services/embedding.ts index a8c37ade97..38793680ed 100644 --- a/web/packages/new/photos/services/embedding.ts +++ b/web/packages/new/photos/services/embedding.ts @@ -84,7 +84,6 @@ type RemoteEmbedding = z.infer; * with remote (See: [Note: Ignoring embeddings for unknown files]). */ export const syncRemoteFaceEmbeddings = async () => { - let sinceTime = faceEmbeddingSyncTime(); // Include files from trash, otherwise they'll get unnecessarily reindexed // if the user restores them from trash before permanent deletion. const localFiles = (await getAllLocalFiles()).concat( @@ -92,6 +91,9 @@ export const syncRemoteFaceEmbeddings = async () => { ); const localFilesByID = new Map(localFiles.map((f) => [f.id, f])); + // Delete embeddings for files which are no longer present locally. + // pruneFaceEmbeddings(localFilesByID); + const decryptEmbedding = async (remoteEmbedding: RemoteEmbedding) => { const file = localFilesByID.get(remoteEmbedding.fileID); // [Note: Ignoring embeddings for unknown files] @@ -130,6 +132,8 @@ export const syncRemoteFaceEmbeddings = async () => { } }; + let sinceTime = faceEmbeddingSyncTime(); + // TODO: eslint has fixed this spurious warning, but we're not on the latest // version yet, so add a disable. // https://github.com/eslint/eslint/pull/18286 diff --git a/web/apps/photos/src/services/face/db.ts b/web/packages/new/photos/services/face/db.ts similarity index 100% rename from web/apps/photos/src/services/face/db.ts rename to web/packages/new/photos/services/face/db.ts diff --git a/web/apps/photos/src/services/face/types.ts b/web/packages/new/photos/services/face/types.ts similarity index 100% rename from web/apps/photos/src/services/face/types.ts rename to web/packages/new/photos/services/face/types.ts From 53452344f3b6845554135435a6944607d9833f69 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 27 Jun 2024 09:28:31 +0530 Subject: [PATCH 16/34] Lints --- web/packages/new/photos/services/face/db.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/web/packages/new/photos/services/face/db.ts b/web/packages/new/photos/services/face/db.ts index 2a75ab7ec4..bd8255f40f 100644 --- a/web/packages/new/photos/services/face/db.ts +++ b/web/packages/new/photos/services/face/db.ts @@ -170,7 +170,7 @@ export const saveFaceIndex = async (faceIndex: FaceIndex) => { const tx = db.transaction(["face-index", "file-status"], "readwrite"); const indexStore = tx.objectStore("face-index"); const statusStore = tx.objectStore("file-status"); - return Promise.all([ + await Promise.all([ indexStore.put(faceIndex), statusStore.put({ fileID: faceIndex.fileID, @@ -178,7 +178,7 @@ export const saveFaceIndex = async (faceIndex: FaceIndex) => { failureCount: 0, }), tx.done, - ]).then(() => {} /* convert result to void */); + ]); }; /** @@ -242,7 +242,7 @@ export const syncWithLocalFiles = async (localFileIDs: number[]) => { const newFileIDs = localFileIDs.filter((id) => !fdb.has(id)); const removedFileIDs = fdbFileIDs.filter((id) => !local.has(id)); - return Promise.all( + await Promise.all( [ newFileIDs.map((id) => tx.objectStore("file-status").put({ @@ -257,7 +257,7 @@ export const syncWithLocalFiles = async (localFileIDs: number[]) => { removedFileIDs.map((id) => tx.objectStore("face-index").delete(id)), tx.done, ].flat(), - ).then(() => {} /* convert result to void */); + ); }; /** From b63a15a5212c1488837f9adf5dab8a7e0c186830 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 27 Jun 2024 09:59:54 +0530 Subject: [PATCH 17/34] Move trash handling to existing sync --- web/apps/photos/src/services/face/indexer.ts | 20 +++++-- .../machineLearning/machineLearningService.ts | 4 +- web/packages/new/photos/services/embedding.ts | 10 ++-- web/packages/new/photos/services/face/db.ts | 54 ++++++++++++++----- 4 files changed, 60 insertions(+), 28 deletions(-) diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index 0f78c20e2e..74e700a3f0 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -2,13 +2,16 @@ import { faceIndex, indexableFileIDs, indexedAndIndexableCounts, - syncWithLocalFiles, + syncAssumingLocalFileIDs, } from "@/new/photos/services/face/db"; import { isBetaUser, isInternalUser, } from "@/new/photos/services/feature-flags"; -import { getAllLocalFiles } from "@/new/photos/services/files"; +import { + getAllLocalFiles, + getLocalTrashedFiles, +} from "@/new/photos/services/files"; import type { EnteFile } from "@/new/photos/types/file"; import { ComlinkWorker } from "@/next/worker/comlink-worker"; import { ensure } from "@/utils/ensure"; @@ -229,13 +232,13 @@ export const setIsFaceIndexingEnabled = async (enabled: boolean) => { * about. Then return the next {@link count} files that still need to be * indexed. * - * For more specifics of what a "sync" entails, see {@link syncWithLocalFiles}. + * For specifics of what a "sync" entails, see {@link syncAssumingLocalFileIDs}. * * @param userID Sync only files owned by a {@link userID} with the face DB. * * @param count Limit the resulting list of indexable files to {@link count}. */ -export const syncAndGetFilesToIndex = async ( +export const syncWithLocalFilesAndGetFilesToIndex = async ( userID: number, count: number, ): Promise => { @@ -246,7 +249,14 @@ export const syncAndGetFilesToIndex = async ( localFiles.filter(isIndexable).map((f) => [f.id, f]), ); - await syncWithLocalFiles([...localFilesByID.keys()]); + const localFilesInTrashIDs = (await getLocalTrashedFiles()).map( + (f) => f.id, + ); + + await syncAssumingLocalFileIDs( + [...localFilesByID.keys()], + localFilesInTrashIDs, + ); const fileIDsToIndex = await indexableFileIDs(count); return fileIDsToIndex.map((id) => ensure(localFilesByID.get(id))); diff --git a/web/apps/photos/src/services/machineLearning/machineLearningService.ts b/web/apps/photos/src/services/machineLearning/machineLearningService.ts index 7dbadf9186..be182219a8 100644 --- a/web/apps/photos/src/services/machineLearning/machineLearningService.ts +++ b/web/apps/photos/src/services/machineLearning/machineLearningService.ts @@ -2,7 +2,7 @@ import { EnteFile } from "@/new/photos/types/file"; import log from "@/next/log"; import { CustomError, parseUploadErrorCodes } from "@ente/shared/error"; import PQueue from "p-queue"; -import { syncAndGetFilesToIndex } from "services/face/indexer"; +import { syncWithLocalFilesAndGetFilesToIndex } from "services/face/indexer"; import { FaceIndexerWorker } from "services/face/indexer.worker"; const batchSize = 200; @@ -56,7 +56,7 @@ class MachineLearningService { const syncContext = await this.getSyncContext(token, userID, userAgent); - syncContext.outOfSyncFiles = await syncAndGetFilesToIndex( + syncContext.outOfSyncFiles = await syncWithLocalFilesAndGetFilesToIndex( userID, batchSize, ); diff --git a/web/packages/new/photos/services/embedding.ts b/web/packages/new/photos/services/embedding.ts index 38793680ed..3c2df24ec3 100644 --- a/web/packages/new/photos/services/embedding.ts +++ b/web/packages/new/photos/services/embedding.ts @@ -75,13 +75,12 @@ const RemoteEmbedding = z.object({ type RemoteEmbedding = z.infer; /** - * Fetch new or updated face embeddings with the server and save them locally. - * Also prune local embeddings for any files no longer exist locally. + * Fetch new or updated face embeddings from remote and save them locally. * * It takes no parameters since it saves the last sync time in local storage. * - * Precondition: This function should be called only after we have synced files - * with remote (See: [Note: Ignoring embeddings for unknown files]). + * This function should be called only after we have synced files with remote. + * See: [Note: Ignoring embeddings for unknown files]. */ export const syncRemoteFaceEmbeddings = async () => { // Include files from trash, otherwise they'll get unnecessarily reindexed @@ -91,9 +90,6 @@ export const syncRemoteFaceEmbeddings = async () => { ); const localFilesByID = new Map(localFiles.map((f) => [f.id, f])); - // Delete embeddings for files which are no longer present locally. - // pruneFaceEmbeddings(localFilesByID); - const decryptEmbedding = async (remoteEmbedding: RemoteEmbedding) => { const file = localFilesByID.get(remoteEmbedding.fileID); // [Note: Ignoring embeddings for unknown files] diff --git a/web/packages/new/photos/services/face/db.ts b/web/packages/new/photos/services/face/db.ts index bd8255f40f..5a86605a45 100644 --- a/web/packages/new/photos/services/face/db.ts +++ b/web/packages/new/photos/services/face/db.ts @@ -217,30 +217,54 @@ export const addFileEntry = async (fileID: number) => { * Sync entries in the face DB to align with the state of local files outside * face DB. * - * @param localFileIDs Local {@link EnteFile}s, keyed by their IDs. These are - * all the files that the client is aware of, filtered to only keep the files - * that the user owns and the formats that can be indexed by our current face - * indexing pipeline. + * @param localFileIDs IDs of all the files that the client is aware of filtered + * to only keep the files that the user owns and the formats that can be indexed + * by our current face indexing pipeline. * - * This function syncs the state of file entries in face DB to the state of file - * entries stored otherwise by the client locally. + * @param localFilesInTrashIDs IDs of all the files in trash. * - * - Files (identified by their ID) that are present locally but are not yet in - * face DB get a fresh entry in face DB (and are marked as indexable). + * This function then updates the state of file entries in face DB to the be in + * "sync" with these provided local file IDS. * - * - Files that are not present locally but still exist in face DB are removed - * from face DB (including its face index, if any). + * - Files that are present locally but are not yet in face DB get a fresh entry + * in face DB (and are marked as indexable). + * + * - Files that are not present locally (nor are in trash) but still exist in + * face DB are removed from face DB (including their face index, if any). + * + * - Files that are not present locally but are in the trash are retained in + * face DB if their status is "indexed" (otherwise they too are removed). This + * is prevent churn (re-indexing) if the user moves some files to trash but + * then later restores them before they get permanently deleted. */ -export const syncWithLocalFiles = async (localFileIDs: number[]) => { +export const syncAssumingLocalFileIDs = async ( + localFileIDs: number[], + localFilesInTrashIDs: number[], +) => { const db = await faceDB(); const tx = db.transaction(["face-index", "file-status"], "readwrite"); const fdbFileIDs = await tx.objectStore("file-status").getAllKeys(); + const fdbIndexedFileIDs = await tx + .objectStore("file-status") + .getAllKeys(IDBKeyRange.only("indexed")); const local = new Set(localFileIDs); + const localTrash = new Set(localFilesInTrashIDs); const fdb = new Set(fdbFileIDs); + const fdbIndexed = new Set(fdbIndexedFileIDs); const newFileIDs = localFileIDs.filter((id) => !fdb.has(id)); - const removedFileIDs = fdbFileIDs.filter((id) => !local.has(id)); + const fileIDsToRemove = fdbFileIDs.filter((id) => { + if (local.has(id)) return false; // Still exists + if (localTrash.has(id)) { + // Exists in trash + if (fdbIndexed.has(id)) { + // But is already indexed, so let it be. + return false; + } + } + return true; // Remove + }); await Promise.all( [ @@ -251,10 +275,12 @@ export const syncWithLocalFiles = async (localFileIDs: number[]) => { failureCount: 0, }), ), - removedFileIDs.map((id) => + fileIDsToRemove.map((id) => tx.objectStore("file-status").delete(id), ), - removedFileIDs.map((id) => tx.objectStore("face-index").delete(id)), + fileIDsToRemove.map((id) => + tx.objectStore("face-index").delete(id), + ), tx.done, ].flat(), ); From 36d8c2a42717fd579c581e70b94ce99316395923 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 27 Jun 2024 10:05:18 +0530 Subject: [PATCH 18/34] Specific-er than sync --- web/packages/new/photos/services/embedding.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/packages/new/photos/services/embedding.ts b/web/packages/new/photos/services/embedding.ts index 3c2df24ec3..a74209837d 100644 --- a/web/packages/new/photos/services/embedding.ts +++ b/web/packages/new/photos/services/embedding.ts @@ -82,7 +82,7 @@ type RemoteEmbedding = z.infer; * This function should be called only after we have synced files with remote. * See: [Note: Ignoring embeddings for unknown files]. */ -export const syncRemoteFaceEmbeddings = async () => { +export const pullRemoteFaceEmbeddings = async () => { // Include files from trash, otherwise they'll get unnecessarily reindexed // if the user restores them from trash before permanent deletion. const localFiles = (await getAllLocalFiles()).concat( From dc5b0b4393bbbe6cfaa89ed129fce4349557598c Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 27 Jun 2024 10:32:38 +0530 Subject: [PATCH 19/34] Save --- web/packages/new/photos/services/embedding.ts | 97 ++++++++----------- 1 file changed, 38 insertions(+), 59 deletions(-) diff --git a/web/packages/new/photos/services/embedding.ts b/web/packages/new/photos/services/embedding.ts index a74209837d..77b0e3dbc1 100644 --- a/web/packages/new/photos/services/embedding.ts +++ b/web/packages/new/photos/services/embedding.ts @@ -5,6 +5,7 @@ import log from "@/next/log"; import { nullToUndefined } from "@/utils/transform"; import { z } from "zod"; import { decryptFileMetadata } from "../../common/crypto/ente"; +import { saveFaceIndex } from "./face/db"; import { getAllLocalFiles } from "./files"; /** @@ -88,46 +89,29 @@ export const pullRemoteFaceEmbeddings = async () => { const localFiles = (await getAllLocalFiles()).concat( await getLocalTrashedFiles(), ); + // [Note: Ignoring embeddings for unknown files] + // + // We need the file to decrypt the embedding. This is easily ensured by + // running the embedding sync after we have synced our local files with + // remote. + // + // Still, it might happen that we come across an embedding for which we + // don't have the corresponding file locally. We can put them in two + // buckets: + // + // 1. Known case: In rare cases we might get a diff entry for an embedding + // corresponding to a file which has been deleted (but whose embedding + // is enqueued for deletion). Client should expect such a scenario, but + // all they have to do is just ignore such embeddings. + // + // 2. Other unknown cases: Even if somehow we end up with an embedding for + // a existent file which we don't have locally, it is fine because the + // current client will just regenerate the embedding if the file really + // exists and gets locally found later. There would be a bit of + // duplicate work, but that's fine as long as there isn't a systematic + // scenario where this happens. const localFilesByID = new Map(localFiles.map((f) => [f.id, f])); - const decryptEmbedding = async (remoteEmbedding: RemoteEmbedding) => { - const file = localFilesByID.get(remoteEmbedding.fileID); - // [Note: Ignoring embeddings for unknown files] - // - // We need the file to decrypt the embedding. This is easily ensured by - // running the embedding sync after we have synced our local files with - // remote. - // - // Still, it might happen that we come across an embedding for which we - // don't have the corresponding file locally. We can put them in two - // buckets: - // - // 1. Known case: In rare cases we might get a diff entry for an - // embedding corresponding to a file which has been deleted (but - // whose embedding is enqueued for deletion). Client should expect - // such a scenario, but all they have to do is just ignore such - // embeddings. - // - // 2. Other unknown cases: Even if somehow we end up with an embedding - // for a existent file which we don't have locally, it is fine - // because the current client will just regenerate the embedding if - // the file really exists and gets locally found later. There would - // be a bit of duplicate work, but that's fine as long as there - // isn't a systematic scenario where this happens. - if (!file) return undefined; - try { - const decryptedString = await decryptFileMetadata( - remoteEmbedding.encryptedEmbedding, - remoteEmbedding.decryptionHeader, - file.key, - ); - return decryptedString; - } catch (e) { - log.warn("Ignoring unparseable embedding", e); - return undefined; - } - }; - let sinceTime = faceEmbeddingSyncTime(); // TODO: eslint has fixed this spurious warning, but we're not on the latest @@ -141,31 +125,26 @@ export const pullRemoteFaceEmbeddings = async () => { sinceTime, ); if (remoteEmbeddings.length == 0) break; - void (await Promise.all(remoteEmbeddings.map(decryptEmbedding))); - sinceTime = remoteEmbeddings.reduce( - (max, { updatedAt }) => Math.max(max, updatedAt), - sinceTime, - ); + for (const remoteEmbedding of remoteEmbeddings) { + sinceTime = Math.max(sinceTime, remoteEmbedding.updatedAt); + try { + const file = localFilesByID.get(remoteEmbedding.fileID); + if (!file) continue; + const jsonString = await decryptFileMetadata( + remoteEmbedding.encryptedEmbedding, + remoteEmbedding.decryptionHeader, + file.key, + ); + const faceIndex = FaceIndex.parse(JSON.parse(jsonString)); + await saveFaceIndex(faceIndex); + } catch (e) { + log.warn("Ignoring unparseable embedding", e); + } + } saveFaceEmbeddingSyncTime(sinceTime); } }; -// const decryptFaceEmbedding = async (remoteEmbedding: RemoteEmbedding) => { -// const fileKey = fileIdToKeyMap.get(embedding.fileID); -// if (!fileKey) { -// throw Error(CustomError.FILE_NOT_FOUND); -// } -// const decryptedData = await worker.decryptMetadata( -// embedding.encryptedEmbedding, -// embedding.decryptionHeader, -// fileIdToKeyMap.get(embedding.fileID), -// ); -// return { -// ...decryptedData, -// updatedAt: embedding.updatedAt, -// } as unknown as FileML; -// }; - /** * The updatedAt of the most recent face {@link RemoteEmbedding} we've retrieved * and saved from remote, or 0. From 84a03dafe35ccf0083fff05c338bc36eb6ca2b76 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 27 Jun 2024 10:41:56 +0530 Subject: [PATCH 20/34] Isolate what changes --- .../photos/src/services/embeddingService.ts | 2 +- web/packages/new/photos/services/embedding.ts | 38 +++++++++---------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/web/apps/photos/src/services/embeddingService.ts b/web/apps/photos/src/services/embeddingService.ts index 859557cbc9..b255086607 100644 --- a/web/apps/photos/src/services/embeddingService.ts +++ b/web/apps/photos/src/services/embeddingService.ts @@ -1,4 +1,5 @@ import type { EmbeddingModel } from "@/new/photos/services/embedding"; +import type { FaceIndex } from "@/new/photos/services/face/types"; import { getAllLocalFiles, getLocalTrashedFiles, @@ -20,7 +21,6 @@ import type { PutEmbeddingRequest, } from "types/embedding"; import { getLocalCollections } from "./collectionService"; -import type { FaceIndex } from "./face/types"; type FileML = FaceIndex & { updatedAt: number; diff --git a/web/packages/new/photos/services/embedding.ts b/web/packages/new/photos/services/embedding.ts index 77b0e3dbc1..c7f30afd8d 100644 --- a/web/packages/new/photos/services/embedding.ts +++ b/web/packages/new/photos/services/embedding.ts @@ -84,6 +84,10 @@ type RemoteEmbedding = z.infer; * See: [Note: Ignoring embeddings for unknown files]. */ export const pullRemoteFaceEmbeddings = async () => { + const model: EmbeddingModel = "file-ml-clip-face"; + const saveEmbedding = (jsonString: string) => + saveFaceIndex(FaceIndex.parse(JSON.parse(jsonString))); + // Include files from trash, otherwise they'll get unnecessarily reindexed // if the user restores them from trash before permanent deletion. const localFiles = (await getAllLocalFiles()).concat( @@ -112,18 +116,14 @@ export const pullRemoteFaceEmbeddings = async () => { // scenario where this happens. const localFilesByID = new Map(localFiles.map((f) => [f.id, f])); - let sinceTime = faceEmbeddingSyncTime(); - + let sinceTime = embeddingSyncTime(model); // TODO: eslint has fixed this spurious warning, but we're not on the latest // version yet, so add a disable. // https://github.com/eslint/eslint/pull/18286 /* eslint-disable no-constant-condition */ // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition while (true) { - const remoteEmbeddings = await getEmbeddingsDiff( - "file-ml-clip-face", - sinceTime, - ); + const remoteEmbeddings = await getEmbeddingsDiff(model, sinceTime); if (remoteEmbeddings.length == 0) break; for (const remoteEmbedding of remoteEmbeddings) { sinceTime = Math.max(sinceTime, remoteEmbedding.updatedAt); @@ -135,29 +135,30 @@ export const pullRemoteFaceEmbeddings = async () => { remoteEmbedding.decryptionHeader, file.key, ); - const faceIndex = FaceIndex.parse(JSON.parse(jsonString)); - await saveFaceIndex(faceIndex); + await saveEmbedding(jsonString); } catch (e) { - log.warn("Ignoring unparseable embedding", e); + log.warn(`Ignoring unparseable ${model} embedding`, e); } } - saveFaceEmbeddingSyncTime(sinceTime); + saveEmbeddingSyncTime(sinceTime, model); } }; /** - * The updatedAt of the most recent face {@link RemoteEmbedding} we've retrieved - * and saved from remote, or 0. + * The updatedAt of the most recent {@link RemoteEmbedding} for {@link model} + * we've retrieved from remote. + * + * Returns 0 if there is no such embedding. * * This value is persisted to local storage. To update it, use - * {@link saveFaceEmbeddingSyncMarker}. + * {@link saveEmbeddingSyncTime}. */ -const faceEmbeddingSyncTime = () => - parseInt(localStorage.getItem("faceEmbeddingSyncTime") ?? "0"); +const embeddingSyncTime = (model: EmbeddingModel) => + parseInt(localStorage.getItem("embeddingSyncTime:" + model) ?? "0"); -/** Sibling of {@link faceEmbeddingSyncMarker}. */ -const saveFaceEmbeddingSyncTime = (t: number) => - localStorage.setItem("faceEmbeddingSyncTime", `${t}`); +/** Sibling of {@link embeddingSyncTime}. */ +const saveEmbeddingSyncTime = (t: number, model: EmbeddingModel) => + localStorage.setItem("embeddingSyncTime:" + model, `${t}`); /** * Zod schemas for the {@link FaceIndex} types. @@ -183,7 +184,6 @@ const saveFaceEmbeddingSyncTime = (t: number) => * out of sync in the shape of the types they represent, the TypeScript compiler * will flag it for us. */ -// eslint-disable-next-line @typescript-eslint/no-unused-vars const FaceIndex = z .object({ fileID: z.number(), From ddf18bd0368bd26174a7051c0de49ea39a0cad61 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 27 Jun 2024 11:21:52 +0530 Subject: [PATCH 21/34] Handle versioning --- web/apps/photos/src/services/face/crop.ts | 2 +- web/apps/photos/src/services/face/f-index.ts | 10 ++++- web/apps/photos/src/services/face/index.ts | 0 .../src/services/face/indexer.worker.ts | 2 +- web/apps/photos/src/services/face/remote.ts | 2 +- web/packages/new/photos/services/embedding.ts | 43 ++++++++++++------- .../new/photos/services/face/types.ts | 5 +++ 7 files changed, 44 insertions(+), 20 deletions(-) delete mode 100644 web/apps/photos/src/services/face/index.ts diff --git a/web/apps/photos/src/services/face/crop.ts b/web/apps/photos/src/services/face/crop.ts index ff1d6026af..ae5d44f104 100644 --- a/web/apps/photos/src/services/face/crop.ts +++ b/web/apps/photos/src/services/face/crop.ts @@ -1,6 +1,6 @@ +import type { Box } from "@/new/photos/services/face/types"; import { blobCache } from "@/next/blob-cache"; import type { FaceAlignment } from "./f-index"; -import type { Box } from "./types"; export const saveFaceCrop = async ( imageBitmap: ImageBitmap, diff --git a/web/apps/photos/src/services/face/f-index.ts b/web/apps/photos/src/services/face/f-index.ts index 584b733148..cf41623e94 100644 --- a/web/apps/photos/src/services/face/f-index.ts +++ b/web/apps/photos/src/services/face/f-index.ts @@ -1,5 +1,12 @@ import { FILE_TYPE } from "@/media/file-type"; import { decodeLivePhoto } from "@/media/live-photo"; +import type { + Box, + Dimensions, + Face, + Point, +} from "@/new/photos/services/face/types"; +import { faceIndexingVersion } from "@/new/photos/services/face/types"; import type { EnteFile } from "@/new/photos/types/file"; import log from "@/next/log"; import { workerBridge } from "@/next/worker/worker-bridge"; @@ -21,7 +28,6 @@ import { pixelRGBBilinear, warpAffineFloat32List, } from "./image"; -import type { Box, Dimensions, Face, Point } from "./types"; /** * Index faces in the given file. @@ -64,7 +70,7 @@ export const indexFaces = async ( width, height, faceEmbedding: { - version: 1, + version: faceIndexingVersion, client: userAgent, faces: await indexFacesInBitmap(fileID, imageBitmap), }, diff --git a/web/apps/photos/src/services/face/index.ts b/web/apps/photos/src/services/face/index.ts deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/web/apps/photos/src/services/face/indexer.worker.ts b/web/apps/photos/src/services/face/indexer.worker.ts index 27608a31c8..9df8a455ef 100644 --- a/web/apps/photos/src/services/face/indexer.worker.ts +++ b/web/apps/photos/src/services/face/indexer.worker.ts @@ -3,12 +3,12 @@ import { markIndexingFailed, saveFaceIndex, } from "@/new/photos/services/face/db"; +import type { FaceIndex } from "@/new/photos/services/face/types"; import type { EnteFile } from "@/new/photos/types/file"; import log from "@/next/log"; import { fileLogID } from "utils/file"; import { indexFaces } from "./f-index"; import { putFaceIndex } from "./remote"; -import type { FaceIndex } from "./types"; /** * Index faces in a file, save the persist the results locally, and put them on diff --git a/web/apps/photos/src/services/face/remote.ts b/web/apps/photos/src/services/face/remote.ts index 2c4209ac28..daf177a0e4 100644 --- a/web/apps/photos/src/services/face/remote.ts +++ b/web/apps/photos/src/services/face/remote.ts @@ -1,8 +1,8 @@ +import type { FaceIndex } from "@/new/photos/services/face/types"; import type { EnteFile } from "@/new/photos/types/file"; import log from "@/next/log"; import ComlinkCryptoWorker from "@ente/shared/crypto"; import { putEmbedding } from "services/embeddingService"; -import type { FaceIndex } from "./types"; export const putFaceIndex = async ( enteFile: EnteFile, diff --git a/web/packages/new/photos/services/embedding.ts b/web/packages/new/photos/services/embedding.ts index c7f30afd8d..83d3b6d71a 100644 --- a/web/packages/new/photos/services/embedding.ts +++ b/web/packages/new/photos/services/embedding.ts @@ -1,11 +1,11 @@ import { getLocalTrashedFiles } from "@/new/photos/services/files"; import { authenticatedRequestHeaders } from "@/next/http"; -import { apiURL } from "@/next/origins"; import log from "@/next/log"; -import { nullToUndefined } from "@/utils/transform"; +import { apiURL } from "@/next/origins"; import { z } from "zod"; import { decryptFileMetadata } from "../../common/crypto/ente"; import { saveFaceIndex } from "./face/db"; +import { faceIndexingVersion, type FaceIndex } from "./face/types"; import { getAllLocalFiles } from "./files"; /** @@ -22,17 +22,19 @@ import { getAllLocalFiles } from "./files"; * * [Note: Handling versioning of embeddings] * - * The embeddings themselves have a version included in them, so it is possible + * The embeddings themselves have a version embedded in them, so it is possible * for us to make backward compatible updates to the indexing process on newer - * clients. + * clients (There is a top level version field too but that is not used. * * If we bump the version of same model (say when indexing on a newer client), * the assumption will be that older client will be able to consume the - * response. Say if we improve blur detection, older client should just consume - * the newer version and not try to index the file locally. + * response. e.g. Say if we improve blur detection, older client should just + * consume embeddings with a newer version and not try to index the file again + * locally. * - * If you get version that is older, client should discard and try to index - * locally (if needed) and also put the newer version it has on remote. + * If we get an embedding with version that is older than the version the client + * supports, then the client should ignore it. This way, the file will get + * reindexed locally an embedding with a newer version will get put to remote. * * In the case where the changes are not backward compatible and can only be * consumed by clients with the relevant scaffolding, then we change this @@ -65,12 +67,6 @@ const RemoteEmbedding = z.object({ decryptionHeader: z.string(), /** Last time (epoch ms) this embedding was updated. */ updatedAt: z.number(), - /** - * The version for the embedding. Optional. - * - * See: [Note: Handling versioning of embeddings] - */ - version: z.number().nullish().transform(nullToUndefined), }); type RemoteEmbedding = z.infer; @@ -86,7 +82,7 @@ type RemoteEmbedding = z.infer; export const pullRemoteFaceEmbeddings = async () => { const model: EmbeddingModel = "file-ml-clip-face"; const saveEmbedding = (jsonString: string) => - saveFaceIndex(FaceIndex.parse(JSON.parse(jsonString))); + saveFaceIndexIfNewer(FaceIndex.parse(JSON.parse(jsonString))); // Include files from trash, otherwise they'll get unnecessarily reindexed // if the user restores them from trash before permanent deletion. @@ -160,6 +156,23 @@ const embeddingSyncTime = (model: EmbeddingModel) => const saveEmbeddingSyncTime = (t: number, model: EmbeddingModel) => localStorage.setItem("embeddingSyncTime:" + model, `${t}`); +/** + * Save the given {@link faceIndex} locally if it is newer than the one we have. + * + * This is a variant of {@link saveFaceIndex} that performs version checking as + * described in [Note: Handling versioning of embeddings]. + */ +export const saveFaceIndexIfNewer = async (index: FaceIndex) => { + const version = index.faceEmbedding.version; + if (version <= faceIndexingVersion) { + log.info( + `Ignoring remote face index with version ${version} not newer than what our indexer supports (${faceIndexingVersion})`, + ); + return; + } + return saveFaceIndex(index); +}; + /** * Zod schemas for the {@link FaceIndex} types. * diff --git a/web/packages/new/photos/services/face/types.ts b/web/packages/new/photos/services/face/types.ts index a1db97a9af..44c78cc838 100644 --- a/web/packages/new/photos/services/face/types.ts +++ b/web/packages/new/photos/services/face/types.ts @@ -1,3 +1,8 @@ +/** The face indexing version supported by the current client. */ +// TODO: This belongs better to f-index.ts, but that file's in a different +// package currently, move it there once these two files are together again. +export const faceIndexingVersion = 1; + /** * The faces in a file (and an embedding for each of them). * From 7d8ade7fe41e82df288fe60e845196ef1b43e733 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 27 Jun 2024 11:31:42 +0530 Subject: [PATCH 22/34] Extract --- web/packages/new/photos/services/embedding.ts | 48 ++++++++++++++----- 1 file changed, 36 insertions(+), 12 deletions(-) diff --git a/web/packages/new/photos/services/embedding.ts b/web/packages/new/photos/services/embedding.ts index 83d3b6d71a..603b14767b 100644 --- a/web/packages/new/photos/services/embedding.ts +++ b/web/packages/new/photos/services/embedding.ts @@ -72,18 +72,25 @@ const RemoteEmbedding = z.object({ type RemoteEmbedding = z.infer; /** - * Fetch new or updated face embeddings from remote and save them locally. + * Fetch new or updated embeddings from remote and save them locally. * - * It takes no parameters since it saves the last sync time in local storage. + * @param model The {@link EmbeddingModel} for which to pull embeddings. For + * each model, this function maintains the last sync time in local storage so + * subsequent fetches only pull what's new. + * + * @param save A function that is called to save the embedding. The save process + * can be model specific, so this provides us a hook to reuse the surrounding + * pull mechanisms while varying the save itself. This function will be passed + * the decrypted embedding string. If it throws, then we'll log about but + * otherwise ignore the embedding under consideration. * * This function should be called only after we have synced files with remote. * See: [Note: Ignoring embeddings for unknown files]. */ -export const pullRemoteFaceEmbeddings = async () => { - const model: EmbeddingModel = "file-ml-clip-face"; - const saveEmbedding = (jsonString: string) => - saveFaceIndexIfNewer(FaceIndex.parse(JSON.parse(jsonString))); - +export const pullEmbeddings = async ( + model: EmbeddingModel, + save: (decryptedEmbedding: string) => Promise, +) => { // Include files from trash, otherwise they'll get unnecessarily reindexed // if the user restores them from trash before permanent deletion. const localFiles = (await getAllLocalFiles()).concat( @@ -119,6 +126,7 @@ export const pullRemoteFaceEmbeddings = async () => { /* eslint-disable no-constant-condition */ // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition while (true) { + let c = 0; const remoteEmbeddings = await getEmbeddingsDiff(model, sinceTime); if (remoteEmbeddings.length == 0) break; for (const remoteEmbedding of remoteEmbeddings) { @@ -126,17 +134,20 @@ export const pullRemoteFaceEmbeddings = async () => { try { const file = localFilesByID.get(remoteEmbedding.fileID); if (!file) continue; - const jsonString = await decryptFileMetadata( - remoteEmbedding.encryptedEmbedding, - remoteEmbedding.decryptionHeader, - file.key, + await save( + await decryptFileMetadata( + remoteEmbedding.encryptedEmbedding, + remoteEmbedding.decryptionHeader, + file.key, + ), ); - await saveEmbedding(jsonString); + c++; } catch (e) { log.warn(`Ignoring unparseable ${model} embedding`, e); } } saveEmbeddingSyncTime(sinceTime, model); + log.info(`Fetched ${c} ${model} embeddings`); } }; @@ -156,6 +167,19 @@ const embeddingSyncTime = (model: EmbeddingModel) => const saveEmbeddingSyncTime = (t: number, model: EmbeddingModel) => localStorage.setItem("embeddingSyncTime:" + model, `${t}`); +/** + * Fetch new or updated face embeddings from remote and save them locally. + * + * It takes no parameters since it saves the last sync time in local storage. + * + * This function should be called only after we have synced files with remote. + * See: [Note: Ignoring embeddings for unknown files]. + */ +export const pullFaceEmbeddings = () => + pullEmbeddings("file-ml-clip-face", (jsonString: string) => + saveFaceIndexIfNewer(FaceIndex.parse(JSON.parse(jsonString))), + ); + /** * Save the given {@link faceIndex} locally if it is newer than the one we have. * From ea7619d40520b330fb8281fb8c61aa74d64b8d61 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 27 Jun 2024 11:51:39 +0530 Subject: [PATCH 23/34] Potential entry point --- web/packages/new/photos/services/embedding.ts | 2 +- .../new/photos/services/face/worker.ts | 22 +++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 web/packages/new/photos/services/face/worker.ts diff --git a/web/packages/new/photos/services/embedding.ts b/web/packages/new/photos/services/embedding.ts index 603b14767b..b7fe375975 100644 --- a/web/packages/new/photos/services/embedding.ts +++ b/web/packages/new/photos/services/embedding.ts @@ -87,7 +87,7 @@ type RemoteEmbedding = z.infer; * This function should be called only after we have synced files with remote. * See: [Note: Ignoring embeddings for unknown files]. */ -export const pullEmbeddings = async ( +const pullEmbeddings = async ( model: EmbeddingModel, save: (decryptedEmbedding: string) => Promise, ) => { diff --git a/web/packages/new/photos/services/face/worker.ts b/web/packages/new/photos/services/face/worker.ts new file mode 100644 index 0000000000..0902bed53b --- /dev/null +++ b/web/packages/new/photos/services/face/worker.ts @@ -0,0 +1,22 @@ +import { pullFaceEmbeddings } from "../embedding"; + +/** + * Run operations related to face indexing and search in a Web Worker. + * + * This is a normal class that is however exposed (via comlink) as a proxy + * running inside a Web Worker. This way, we do not bother the main thread with + * tasks that might degrade interactivity. + */ +export class FaceWorker { + private isSyncing = false; + + /** + * Pull embeddings from remote, and start backfilling if needed. + */ + async sync() { + if (this.isSyncing) return; + this.isSyncing = true; + await pullFaceEmbeddings(); + this.isSyncing = false; + } +} From dd3243492a701495d3204595f436b7ac20728f06 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 27 Jun 2024 12:40:48 +0530 Subject: [PATCH 24/34] Use module state --- .../new/photos/services/face/index.ts | 36 +++++++++++++++++++ web/packages/next/worker/comlink-worker.ts | 2 +- 2 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 web/packages/new/photos/services/face/index.ts diff --git a/web/packages/new/photos/services/face/index.ts b/web/packages/new/photos/services/face/index.ts new file mode 100644 index 0000000000..50560db93c --- /dev/null +++ b/web/packages/new/photos/services/face/index.ts @@ -0,0 +1,36 @@ +/** + * @file Main thread interface to {@link FaceWorker}. + */ + +import { ComlinkWorker } from "@/next/worker/comlink-worker"; +import { FaceWorker } from "./worker"; + +/** Cached instance of the {@link ComlinkWorker} that wraps our web worker. */ +let _comlinkWorker: ComlinkWorker | undefined; + +/** Lazily created, cached, instance of {@link FaceWorker}. */ +export const faceWorker = async () => { + let comlinkWorker = _comlinkWorker; + if (!comlinkWorker) _comlinkWorker = comlinkWorker = createComlinkWorker(); + return await comlinkWorker.remote; +}; + +const createComlinkWorker = () => + new ComlinkWorker( + "face", + new Worker(new URL("worker.ts", import.meta.url)), + ); + +/** + * Terminate {@link faceWorker} (if any). + * + * This is useful during logout to immediately stop any background face related + * operations that are in-flight for the current user. After the user logs in + * again, a new {@link faceWorker} will be created on demand. + */ +export const terminateFaceWorker = () => { + if (_comlinkWorker) { + _comlinkWorker.terminate(); + _comlinkWorker = undefined; + } +}; diff --git a/web/packages/next/worker/comlink-worker.ts b/web/packages/next/worker/comlink-worker.ts index a546409839..1a6645d334 100644 --- a/web/packages/next/worker/comlink-worker.ts +++ b/web/packages/next/worker/comlink-worker.ts @@ -25,7 +25,7 @@ export class ComlinkWorker InstanceType> { public terminate() { this.worker.terminate(); - log.debug(() => `Terminated ${this.name}`); + log.debug(() => `Terminated web worker ${this.name}`); } } From 7d46de139e0a57849b7a37e5402f87d554b439f5 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 27 Jun 2024 13:16:09 +0530 Subject: [PATCH 25/34] Logout --- web/apps/photos/src/services/logout.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/web/apps/photos/src/services/logout.ts b/web/apps/photos/src/services/logout.ts index 1a6c5547e8..266247ca98 100644 --- a/web/apps/photos/src/services/logout.ts +++ b/web/apps/photos/src/services/logout.ts @@ -1,3 +1,4 @@ +import { terminateFaceWorker } from "@/new/photos/services/face"; import { clearFaceData } from "@/new/photos/services/face/db"; import { clearFeatureFlagSessionState } from "@/new/photos/services/feature-flags"; import log from "@/next/log"; @@ -41,6 +42,12 @@ export const photosLogout = async () => { ignoreError("CLIP", e); } + try { + terminateFaceWorker(); + } catch (e) { + ignoreError("face", e); + } + const electron = globalThis.electron; if (electron) { try { From 90c15774d7d21f4b0b6e6ce39d18884047a617ee Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 27 Jun 2024 13:28:35 +0530 Subject: [PATCH 26/34] Extract --- web/apps/photos/src/pages/gallery/index.tsx | 20 +++----------- web/apps/photos/src/services/sync.ts | 30 +++++++++++++++++++++ 2 files changed, 33 insertions(+), 17 deletions(-) create mode 100644 web/apps/photos/src/services/sync.ts diff --git a/web/apps/photos/src/pages/gallery/index.tsx b/web/apps/photos/src/pages/gallery/index.tsx index a5c180c3aa..8bac954cbb 100644 --- a/web/apps/photos/src/pages/gallery/index.tsx +++ b/web/apps/photos/src/pages/gallery/index.tsx @@ -1,6 +1,5 @@ import { WhatsNew } from "@/new/photos/components/WhatsNew"; import { shouldShowWhatsNew } from "@/new/photos/services/changelog"; -import { fetchAndSaveFeatureFlagsIfNeeded } from "@/new/photos/services/feature-flags"; import { getLocalFiles, getLocalTrashedFiles, @@ -94,13 +93,12 @@ import { getSectionSummaries, } from "services/collectionService"; import downloadManager from "services/download"; -import { syncCLIPEmbeddings } from "services/embeddingService"; -import { syncEntities } from "services/entityService"; import { syncFiles } from "services/fileService"; import locationSearchService from "services/locationSearchService"; +import { sync } from "services/sync"; import { syncTrash } from "services/trashService"; import uploadManager from "services/upload/uploadManager"; -import { isTokenValid, syncMapEnabled } from "services/userService"; +import { isTokenValid } from "services/userService"; import { Collection, CollectionSummaries } from "types/collection"; import { GalleryContextType, @@ -720,19 +718,7 @@ export default function Gallery() { await syncFiles("normal", normalCollections, setFiles); await syncFiles("hidden", hiddenCollections, setHiddenFiles); await syncTrash(collections, setTrashedFiles); - await syncEntities(); - await syncMapEnabled(); - fetchAndSaveFeatureFlagsIfNeeded(); - const electron = globalThis.electron; - if (electron) { - await syncCLIPEmbeddings(); - // TODO-ML(MR): Disable fetch until we start storing it in the - // same place as the local ones. - // if (isFaceIndexingEnabled()) await syncFaceEmbeddings(); - } - if (clipService.isPlatformSupported()) { - void clipService.scheduleImageEmbeddingExtraction(); - } + await sync(); } catch (e) { switch (e.message) { case CustomError.SESSION_EXPIRED: diff --git a/web/apps/photos/src/services/sync.ts b/web/apps/photos/src/services/sync.ts new file mode 100644 index 0000000000..19f5812834 --- /dev/null +++ b/web/apps/photos/src/services/sync.ts @@ -0,0 +1,30 @@ +import { fetchAndSaveFeatureFlagsIfNeeded } from "@/new/photos/services/feature-flags"; +import { clipService } from "services/clip-service"; +import { syncCLIPEmbeddings } from "services/embeddingService"; +import { syncEntities } from "services/entityService"; +import { syncMapEnabled } from "services/userService"; + +/** + * Perform a soft "refresh" by making various API calls to fetch state from + * remote, using it to update our local state, and triggering periodic jobs that + * depend on the local state. + */ +export const sync = async () => { + // TODO: This is called after we've synced the local files DBs with remote. + // That code belongs here, but currently that state is persisted in the top + // level gallery React component. + + await syncEntities(); + await syncMapEnabled(); + fetchAndSaveFeatureFlagsIfNeeded(); + const electron = globalThis.electron; + if (electron) { + await syncCLIPEmbeddings(); + // TODO-ML(MR): Disable fetch until we start storing it in the + // same place as the local ones. + // if (isFaceIndexingEnabled()) await syncFaceEmbeddings(); + } + if (clipService.isPlatformSupported()) { + void clipService.scheduleImageEmbeddingExtraction(); + } +}; From 57a587301bc1e5cce1070228ddcdd905223a8218 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 27 Jun 2024 13:35:06 +0530 Subject: [PATCH 27/34] Hook back into the app --- web/apps/photos/src/pages/_app.tsx | 4 ++-- web/apps/photos/src/services/face/indexer.ts | 13 ++++++------- web/apps/photos/src/services/searchService.ts | 4 +--- web/apps/photos/src/services/sync.ts | 6 +++--- 4 files changed, 12 insertions(+), 15 deletions(-) diff --git a/web/apps/photos/src/pages/_app.tsx b/web/apps/photos/src/pages/_app.tsx index 1cdcb6b3b8..a012881fb9 100644 --- a/web/apps/photos/src/pages/_app.tsx +++ b/web/apps/photos/src/pages/_app.tsx @@ -202,7 +202,7 @@ export default function App({ Component, pageProps }: AppProps) { } const loadMlSearchState = async () => { try { - const enabled = await isFaceIndexingEnabled(); + const enabled = isFaceIndexingEnabled(); setMlSearchEnabled(enabled); mlWorkManager.setMlSearchEnabled(enabled); } catch (e) { @@ -302,7 +302,7 @@ export default function App({ Component, pageProps }: AppProps) { const showNavBar = (show: boolean) => setShowNavBar(show); const updateMlSearchEnabled = async (enabled: boolean) => { try { - await setIsFaceIndexingEnabled(enabled); + setIsFaceIndexingEnabled(enabled); setMlSearchEnabled(enabled); mlWorkManager.setMlSearchEnabled(enabled); } catch (e) { diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index 74e700a3f0..820b9c4519 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -215,17 +215,16 @@ export const canEnableFaceIndexing = async () => * on any client. This {@link isFaceIndexingEnabled} property, on the other * hand, denotes whether or not indexing is enabled on the current client. */ -export const isFaceIndexingEnabled = async () => { - return localStorage.getItem("faceIndexingEnabled") == "1"; -}; +export const isFaceIndexingEnabled = () => + localStorage.getItem("faceIndexingEnabled") == "1"; /** * Update the (locally stored) value of {@link isFaceIndexingEnabled}. */ -export const setIsFaceIndexingEnabled = async (enabled: boolean) => { - if (enabled) localStorage.setItem("faceIndexingEnabled", "1"); - else localStorage.removeItem("faceIndexingEnabled"); -}; +export const setIsFaceIndexingEnabled = (enabled: boolean) => + enabled + ? localStorage.setItem("faceIndexingEnabled", "1") + : localStorage.removeItem("faceIndexingEnabled"); /** * Sync face DB with the local (and potentially indexable) files that we know diff --git a/web/apps/photos/src/services/searchService.ts b/web/apps/photos/src/services/searchService.ts index 9fe1580e55..5247ef61e4 100644 --- a/web/apps/photos/src/services/searchService.ts +++ b/web/apps/photos/src/services/searchService.ts @@ -32,9 +32,7 @@ export const getDefaultOptions = async () => { return [ // TODO-ML(MR): Skip this for now if indexing is disabled (eventually // the indexing status should not be tied to results). - ...((await isFaceIndexingEnabled()) - ? [await getIndexStatusSuggestion()] - : []), + ...(isFaceIndexingEnabled() ? [await getIndexStatusSuggestion()] : []), ...(await convertSuggestionsToOptions(await getAllPeopleSuggestion())), ].filter((t) => !!t); }; diff --git a/web/apps/photos/src/services/sync.ts b/web/apps/photos/src/services/sync.ts index 19f5812834..05481d9597 100644 --- a/web/apps/photos/src/services/sync.ts +++ b/web/apps/photos/src/services/sync.ts @@ -1,8 +1,10 @@ +import { faceWorker } from "@/new/photos/services/face"; import { fetchAndSaveFeatureFlagsIfNeeded } from "@/new/photos/services/feature-flags"; import { clipService } from "services/clip-service"; import { syncCLIPEmbeddings } from "services/embeddingService"; import { syncEntities } from "services/entityService"; import { syncMapEnabled } from "services/userService"; +import { isFaceIndexingEnabled } from "./face/indexer"; /** * Perform a soft "refresh" by making various API calls to fetch state from @@ -20,9 +22,7 @@ export const sync = async () => { const electron = globalThis.electron; if (electron) { await syncCLIPEmbeddings(); - // TODO-ML(MR): Disable fetch until we start storing it in the - // same place as the local ones. - // if (isFaceIndexingEnabled()) await syncFaceEmbeddings(); + if (isFaceIndexingEnabled()) await (await faceWorker()).sync(); } if (clipService.isPlatformSupported()) { void clipService.scheduleImageEmbeddingExtraction(); From 266796f6194d3adee62877726104e42246f8e6dd Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 29 Jun 2024 10:52:34 +0530 Subject: [PATCH 28/34] wa --- web/packages/new/photos/services/embedding.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/web/packages/new/photos/services/embedding.ts b/web/packages/new/photos/services/embedding.ts index b7fe375975..ec4b84bf7f 100644 --- a/web/packages/new/photos/services/embedding.ts +++ b/web/packages/new/photos/services/embedding.ts @@ -177,6 +177,10 @@ const saveEmbeddingSyncTime = (t: number, model: EmbeddingModel) => */ export const pullFaceEmbeddings = () => pullEmbeddings("file-ml-clip-face", (jsonString: string) => + // eslint-disable-next-line @typescript-eslint/prefer-ts-expect-error, @typescript-eslint/ban-ts-comment + // @ts-ignore TODO: There is no error here, but this file is imported by + // one of our packages that doesn't have strict mode enabled yet, + // causing a spurious error to be emitted in that context. saveFaceIndexIfNewer(FaceIndex.parse(JSON.parse(jsonString))), ); From 4d41f2d64c432ec88d388413f9b224011a0da3ca Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 29 Jun 2024 11:15:43 +0530 Subject: [PATCH 29/34] Remove unused --- web/packages/shared/storage/localStorage/helpers.ts | 8 -------- web/packages/shared/storage/localStorage/index.ts | 1 - 2 files changed, 9 deletions(-) diff --git a/web/packages/shared/storage/localStorage/helpers.ts b/web/packages/shared/storage/localStorage/helpers.ts index 95ae280e37..390ea93d92 100644 --- a/web/packages/shared/storage/localStorage/helpers.ts +++ b/web/packages/shared/storage/localStorage/helpers.ts @@ -21,14 +21,6 @@ export function setJustSignedUp(status: boolean) { setData(LS_KEYS.JUST_SIGNED_UP, { status }); } -export function getLivePhotoInfoShownCount() { - return getData(LS_KEYS.LIVE_PHOTO_INFO_SHOWN_COUNT)?.count ?? 0; -} - -export function setLivePhotoInfoShownCount(count: boolean) { - setData(LS_KEYS.LIVE_PHOTO_INFO_SHOWN_COUNT, { count }); -} - export function getLocalMapEnabled(): boolean { return getData(LS_KEYS.MAP_ENABLED)?.value ?? false; } diff --git a/web/packages/shared/storage/localStorage/index.ts b/web/packages/shared/storage/localStorage/index.ts index 3df5caf973..c95fa01bf7 100644 --- a/web/packages/shared/storage/localStorage/index.ts +++ b/web/packages/shared/storage/localStorage/index.ts @@ -12,7 +12,6 @@ export enum LS_KEYS { SHOW_BACK_BUTTON = "showBackButton", EXPORT = "export", THUMBNAIL_FIX_STATE = "thumbnailFixState", - LIVE_PHOTO_INFO_SHOWN_COUNT = "livePhotoInfoShownCount", // LOGS = "logs", USER_DETAILS = "userDetails", COLLECTION_SORT_BY = "collectionSortBy", From ea51cdfc7744ead266eb19173053eb3c2b8ead52 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 29 Jun 2024 11:17:31 +0530 Subject: [PATCH 30/34] Remove unused --- web/packages/shared/storage/localStorage/index.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/web/packages/shared/storage/localStorage/index.ts b/web/packages/shared/storage/localStorage/index.ts index c95fa01bf7..5a22e9db21 100644 --- a/web/packages/shared/storage/localStorage/index.ts +++ b/web/packages/shared/storage/localStorage/index.ts @@ -2,7 +2,6 @@ import log from "@/next/log"; export enum LS_KEYS { USER = "user", - SESSION = "session", KEY_ATTRIBUTES = "keyAttributes", ORIGINAL_KEY_ATTRIBUTES = "originalKeyAttributes", SUBSCRIPTION = "subscription", @@ -11,12 +10,10 @@ export enum LS_KEYS { JUST_SIGNED_UP = "justSignedUp", SHOW_BACK_BUTTON = "showBackButton", EXPORT = "export", - THUMBNAIL_FIX_STATE = "thumbnailFixState", // LOGS = "logs", USER_DETAILS = "userDetails", COLLECTION_SORT_BY = "collectionSortBy", THEME = "theme", - WAIT_TIME = "waitTime", // Moved to the new wrapper @/next/local-storage // LOCALE = 'locale', MAP_ENABLED = "mapEnabled", From 66cb95e32cfca5ff5f70e8b1c140b8db79557b40 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 29 Jun 2024 11:31:43 +0530 Subject: [PATCH 31/34] Fix hanging paren --- web/packages/new/photos/services/embedding.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/packages/new/photos/services/embedding.ts b/web/packages/new/photos/services/embedding.ts index ec4b84bf7f..2d6e939b5f 100644 --- a/web/packages/new/photos/services/embedding.ts +++ b/web/packages/new/photos/services/embedding.ts @@ -24,7 +24,7 @@ import { getAllLocalFiles } from "./files"; * * The embeddings themselves have a version embedded in them, so it is possible * for us to make backward compatible updates to the indexing process on newer - * clients (There is a top level version field too but that is not used. + * clients (There is also a top level version field too but that is not used). * * If we bump the version of same model (say when indexing on a newer client), * the assumption will be that older client will be able to consume the From bb37630bae05a4285b6a62114d21dca39e0bc34e Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 29 Jun 2024 11:32:21 +0530 Subject: [PATCH 32/34] Nicer --- web/packages/new/photos/services/embedding.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/web/packages/new/photos/services/embedding.ts b/web/packages/new/photos/services/embedding.ts index 2d6e939b5f..82de19ec9d 100644 --- a/web/packages/new/photos/services/embedding.ts +++ b/web/packages/new/photos/services/embedding.ts @@ -126,9 +126,9 @@ const pullEmbeddings = async ( /* eslint-disable no-constant-condition */ // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition while (true) { - let c = 0; const remoteEmbeddings = await getEmbeddingsDiff(model, sinceTime); if (remoteEmbeddings.length == 0) break; + let count = 0; for (const remoteEmbedding of remoteEmbeddings) { sinceTime = Math.max(sinceTime, remoteEmbedding.updatedAt); try { @@ -141,13 +141,13 @@ const pullEmbeddings = async ( file.key, ), ); - c++; + count++; } catch (e) { log.warn(`Ignoring unparseable ${model} embedding`, e); } } saveEmbeddingSyncTime(sinceTime, model); - log.info(`Fetched ${c} ${model} embeddings`); + log.info(`Fetched ${count} ${model} embeddings`); } }; From 8b16b4632f35fa601b52c4af75ca6811a617bd6e Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 29 Jun 2024 11:34:50 +0530 Subject: [PATCH 33/34] Renames --- web/apps/photos/src/services/face/indexer.ts | 12 ++++----- web/packages/new/photos/services/face/db.ts | 26 +++++++++----------- 2 files changed, 17 insertions(+), 21 deletions(-) diff --git a/web/apps/photos/src/services/face/indexer.ts b/web/apps/photos/src/services/face/indexer.ts index 820b9c4519..a7dfe319b9 100644 --- a/web/apps/photos/src/services/face/indexer.ts +++ b/web/apps/photos/src/services/face/indexer.ts @@ -2,7 +2,7 @@ import { faceIndex, indexableFileIDs, indexedAndIndexableCounts, - syncAssumingLocalFileIDs, + updateAssumingLocalFiles, } from "@/new/photos/services/face/db"; import { isBetaUser, @@ -231,7 +231,7 @@ export const setIsFaceIndexingEnabled = (enabled: boolean) => * about. Then return the next {@link count} files that still need to be * indexed. * - * For specifics of what a "sync" entails, see {@link syncAssumingLocalFileIDs}. + * For specifics of what a "sync" entails, see {@link updateAssumingLocalFiles}. * * @param userID Sync only files owned by a {@link userID} with the face DB. * @@ -248,13 +248,11 @@ export const syncWithLocalFilesAndGetFilesToIndex = async ( localFiles.filter(isIndexable).map((f) => [f.id, f]), ); - const localFilesInTrashIDs = (await getLocalTrashedFiles()).map( - (f) => f.id, - ); + const localTrashFileIDs = (await getLocalTrashedFiles()).map((f) => f.id); - await syncAssumingLocalFileIDs( + await updateAssumingLocalFiles( [...localFilesByID.keys()], - localFilesInTrashIDs, + localTrashFileIDs, ); const fileIDsToIndex = await indexableFileIDs(count); diff --git a/web/packages/new/photos/services/face/db.ts b/web/packages/new/photos/services/face/db.ts index 5a86605a45..4e3eb74dab 100644 --- a/web/packages/new/photos/services/face/db.ts +++ b/web/packages/new/photos/services/face/db.ts @@ -214,17 +214,17 @@ export const addFileEntry = async (fileID: number) => { }; /** - * Sync entries in the face DB to align with the state of local files outside + * Update entries in the face DB to align with the state of local files outside * face DB. * * @param localFileIDs IDs of all the files that the client is aware of filtered * to only keep the files that the user owns and the formats that can be indexed * by our current face indexing pipeline. * - * @param localFilesInTrashIDs IDs of all the files in trash. + * @param localTrashFilesIDs IDs of all the files in trash. * * This function then updates the state of file entries in face DB to the be in - * "sync" with these provided local file IDS. + * sync with these provided local file IDS. * * - Files that are present locally but are not yet in face DB get a fresh entry * in face DB (and are marked as indexable). @@ -237,9 +237,9 @@ export const addFileEntry = async (fileID: number) => { * is prevent churn (re-indexing) if the user moves some files to trash but * then later restores them before they get permanently deleted. */ -export const syncAssumingLocalFileIDs = async ( +export const updateAssumingLocalFiles = async ( localFileIDs: number[], - localFilesInTrashIDs: number[], + localTrashFilesIDs: number[], ) => { const db = await faceDB(); const tx = db.transaction(["face-index", "file-status"], "readwrite"); @@ -249,21 +249,21 @@ export const syncAssumingLocalFileIDs = async ( .getAllKeys(IDBKeyRange.only("indexed")); const local = new Set(localFileIDs); - const localTrash = new Set(localFilesInTrashIDs); + const localTrash = new Set(localTrashFilesIDs); const fdb = new Set(fdbFileIDs); const fdbIndexed = new Set(fdbIndexedFileIDs); const newFileIDs = localFileIDs.filter((id) => !fdb.has(id)); - const fileIDsToRemove = fdbFileIDs.filter((id) => { - if (local.has(id)) return false; // Still exists + const removedFileIDs = fdbFileIDs.filter((id) => { + if (local.has(id)) return false; // Still exists. if (localTrash.has(id)) { - // Exists in trash + // Exists in trash. if (fdbIndexed.has(id)) { // But is already indexed, so let it be. return false; } } - return true; // Remove + return true; // Remove. }); await Promise.all( @@ -275,12 +275,10 @@ export const syncAssumingLocalFileIDs = async ( failureCount: 0, }), ), - fileIDsToRemove.map((id) => + removedFileIDs.map((id) => tx.objectStore("file-status").delete(id), ), - fileIDsToRemove.map((id) => - tx.objectStore("face-index").delete(id), - ), + removedFileIDs.map((id) => tx.objectStore("face-index").delete(id)), tx.done, ].flat(), ); From ad2dabcc962c98a2b87cb78658159523ca1c82ff Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 29 Jun 2024 11:43:15 +0530 Subject: [PATCH 34/34] Fix again --- web/packages/next/origins.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/web/packages/next/origins.ts b/web/packages/next/origins.ts index 8200df95a3..13937f42c0 100644 --- a/web/packages/next/origins.ts +++ b/web/packages/next/origins.ts @@ -1,4 +1,5 @@ import { getKV, setKV } from "@/next/kv"; +import { inWorker } from "./env"; /** * Return the origin (scheme, host, port triple) that should be used for making @@ -36,7 +37,7 @@ export const apiURL = async (path: string) => (await apiOrigin()) + path; */ export const customAPIOrigin = async () => { let origin = await getKV("apiOrigin"); - if (!origin) { + if (!origin && !inWorker()) { // TODO: Migration of apiOrigin from local storage to indexed DB // Remove me after a bit (27 June 2024). const legacyOrigin = localStorage.getItem("apiOrigin");