From 4edaebe05453d2dfabb3cb82cdcad3a5427be904 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 4 Sep 2024 15:56:00 +0530 Subject: [PATCH 01/42] Split --- .../new/photos/services/ml/cgroups.ts | 132 ++++++++++++++++++ .../new/photos/services/ml/cluster.ts | 116 +-------------- web/packages/new/photos/services/ml/db.ts | 3 +- .../new/photos/services/user-entity.ts | 4 +- 4 files changed, 138 insertions(+), 117 deletions(-) create mode 100644 web/packages/new/photos/services/ml/cgroups.ts diff --git a/web/packages/new/photos/services/ml/cgroups.ts b/web/packages/new/photos/services/ml/cgroups.ts new file mode 100644 index 0000000000..42274a7e37 --- /dev/null +++ b/web/packages/new/photos/services/ml/cgroups.ts @@ -0,0 +1,132 @@ +/** + * A cgroup ("cluster group") is a group of clusters (possibly containing a + * single cluster) that the user has interacted with. + * + * Interactions include hiding, merging and giving a name and/or a cover photo. + * + * The most frequent interaction is naming a {@link FaceCluster}, which promotes + * it to a become a {@link CGroup}. The promotion comes with the ability to be + * synced with remote (as a "cgroup" user entity). + * + * There after, the user may attach more clusters to the same {@link CGroup}. + * + * > A named cluster group can be thought of as a "person", though this is not + * > necessarily an accurate characterization. e.g. there can be a named cluster + * > group that contains face clusters of pets. + * + * The other form of interaction is hiding. The user may hide a single (unnamed) + * cluster, or they may hide an named {@link CGroup}. In both cases, we promote + * the cluster to a CGroup if needed so that their request to hide gets synced. + * + * While in our local representation we separately maintain clusters and link to + * them from within CGroups by their clusterID, in the remote representation + * clusters themselves don't get synced. Instead, the "cgroup" entities synced + * with remote contain the clusters within themselves. So a group that gets + * synced with remote looks something like: + * + * { id, name, clusters: [{ clusterID, faceIDs }] } + * + */ +export interface CGroup { + /** + * A nanoid for this cluster group. + * + * This is the ID of the "cgroup" user entity (the envelope), and it is not + * contained as part of the group entity payload itself. + */ + id: string; + /** + * A name assigned by the user to this cluster group. + * + * The client should handle both empty strings and undefined as indicating a + * cgroup without a name. When the client needs to set this to an "empty" + * value, which happens when hiding an unnamed cluster, it should it to an + * empty string. That is, expect `"" | undefined`, but set `""`. + */ + name: string | undefined; + /** + * An unordered set of ids of the clusters that belong to this group. + * + * For ergonomics of transportation and persistence this is an array, but it + * should conceptually be thought of as a set. + */ + clusterIDs: string[]; + /** + * True if this cluster group should be hidden. + * + * The user can hide both named cluster groups and single unnamed clusters. + * If the user hides a single cluster that was offered as a suggestion to + * them on a client, the client will create a new unnamed cgroup containing + * it, and set its hidden flag to sync it with remote (so that other clients + * can also stop showing this cluster). + */ + isHidden: boolean; + /** + * The ID of the face that should be used as the cover photo for this + * cluster group (if the user has set one). + * + * This is similar to the [@link displayFaceID}, the difference being: + * + * - {@link avatarFaceID} is the face selected by the user. + * + * - {@link displayFaceID} is the automatic placeholder, and only comes + * into effect if the user has not explicitly selected a face. + */ + avatarFaceID: string | undefined; + /** + * Locally determined ID of the "best" face that should be used as the + * display face, to represent this cluster group in the UI. + * + * This property is not synced with remote. For more details, see + * {@link avatarFaceID}. + */ + displayFaceID: string | undefined; +} + +/** + * Syncronize the user's cluster groups with remote, running local clustering if + * needed. + * + * A cgroup (cluster group) consists of clusters, each of which itself is a set + * of faces. + * + * cgroup << cluster << face + * + * CGroups are synced with remote, while clusters are a local only (though the + * clusters that are part of a cgroup do get synced with remote). + * + * Clusters are generated locally using {@link clusterFaces} function. These + * generated clusters are then mapped to cgroups based on various user actions: + * + * - The user can provide a name for a cluster ("name a person"). This + * upgrades a cluster into a cgroup, and it then gets synced via remote to + * the user's other clients. + * + * - They can attach more clusters to a cgroup ("merge clusters"). + * + * - They can remove a cluster from a cgroup ("break clusters"). + * + * - They can hide a cluster. This creates an unnamed cgroup so that the + * user's other clients know not to show it. + */ +export const syncCGroups = () => { + // 1. Fetch existing cgroups for the user from remote. + // 2. Save them to DB. + // 3. Prune stale faceIDs from the clusters in the DB. + // 4. Rerun clustering using the cgroups and clusters in DB. + // 5. Save the generated clusters to DB. + // + // The user can see both the cgroups and clusters in the UI, but only the + // cgroups are synced. + /* + * After clustering, we also do some routine cleanup. Faces belonging to files + * that have been deleted (including those in Trash) should be pruned off. + * + * We should not make strict assumptions about the clusters we get from remote. + * In particular, the same face ID can be in different clusters. In such cases + * we should assign it arbitrarily assign it to the last cluster we find it in. + * Such leeway is intentionally provided to allow clients some slack in how they + * implement the sync without needing to make an blocking API request for every + * user interaction. + */ +}; diff --git a/web/packages/new/photos/services/ml/cluster.ts b/web/packages/new/photos/services/ml/cluster.ts index e4928aa991..e00c084fef 100644 --- a/web/packages/new/photos/services/ml/cluster.ts +++ b/web/packages/new/photos/services/ml/cluster.ts @@ -3,6 +3,7 @@ import { newNonSecureID } from "@/base/id-worker"; import log from "@/base/log"; import { ensure } from "@/utils/ensure"; import type { EnteFile } from "../../types/file"; +import type { CGroup } from "./cgroups"; import { faceDirection, type Face, type FaceIndex } from "./face"; import { dotProduct } from "./math"; @@ -28,91 +29,6 @@ export interface FaceCluster { faceIDs: string[]; } -/** - * A cgroup ("cluster group") is a group of clusters (possibly containing a - * single cluster) that the user has interacted with. - * - * Interactions include hiding, merging and giving a name and/or a cover photo. - * - * The most frequent interaction is naming a {@link FaceCluster}, which promotes - * it to a become a {@link CGroup}. The promotion comes with the ability to be - * synced with remote (as a "cgroup" user entity). - * - * There after, the user may attach more clusters to the same {@link CGroup}. - * - * > A named cluster group can be thought of as a "person", though this is not - * > necessarily an accurate characterization. e.g. there can be a named cluster - * > group that contains face clusters of pets. - * - * The other form of interaction is hiding. The user may hide a single (unnamed) - * cluster, or they may hide an named {@link CGroup}. In both cases, we promote - * the cluster to a CGroup if needed so that their request to hide gets synced. - * - * While in our local representation we separately maintain clusters and link to - * them from within CGroups by their clusterID, in the remote representation - * clusters themselves don't get synced. Instead, the "cgroup" entities synced - * with remote contain the clusters within themselves. So a group that gets - * synced with remote looks something like: - * - * { id, name, clusters: [{ clusterID, faceIDs }] } - * - */ -export interface CGroup { - /** - * A nanoid for this cluster group. - * - * This is the ID of the "cgroup" user entity (the envelope), and it is not - * contained as part of the group entity payload itself. - */ - id: string; - /** - * A name assigned by the user to this cluster group. - * - * The client should handle both empty strings and undefined as indicating a - * cgroup without a name. When the client needs to set this to an "empty" - * value, which happens when hiding an unnamed cluster, it should it to an - * empty string. That is, expect `"" | undefined`, but set `""`. - */ - name: string | undefined; - /** - * An unordered set of ids of the clusters that belong to this group. - * - * For ergonomics of transportation and persistence this is an array, but it - * should conceptually be thought of as a set. - */ - clusterIDs: string[]; - /** - * True if this cluster group should be hidden. - * - * The user can hide both named cluster groups and single unnamed clusters. - * If the user hides a single cluster that was offered as a suggestion to - * them on a client, the client will create a new unnamed cgroup containing - * it, and set its hidden flag to sync it with remote (so that other clients - * can also stop showing this cluster). - */ - isHidden: boolean; - /** - * The ID of the face that should be used as the cover photo for this - * cluster group (if the user has set one). - * - * This is similar to the [@link displayFaceID}, the difference being: - * - * - {@link avatarFaceID} is the face selected by the user. - * - * - {@link displayFaceID} is the automatic placeholder, and only comes - * into effect if the user has not explicitly selected a face. - */ - avatarFaceID: string | undefined; - /** - * Locally determined ID of the "best" face that should be used as the - * display face, to represent this cluster group in the UI. - * - * This property is not synced with remote. For more details, see - * {@link avatarFaceID}. - */ - displayFaceID: string | undefined; -} - export interface ClusteringOpts { minBlur: number; minScore: number; @@ -149,37 +65,9 @@ export interface ClusterPreviewFace { } /** - * Cluster faces into groups. - * - * A cgroup (cluster group) consists of clusters, each of which itself is a set - * of faces. - * - * cgroup << cluster << face - * - * This function generates clusters locally using a batched form of linear + * Generates clusters from the given faces using a batched form of linear * clustering, with a bit of lookback (and a dollop of heuristics) to get the * clusters to merge across batches. - * - * This user can later tweak these clusters by performing the following actions - * to the list of clusters that they can see: - * - * - They can provide a name for a cluster ("name a person"). This upgrades a - * cluster into a "cgroup", which is an entity that gets synced via remote - * to the user's other clients. - * - * - They can attach more clusters to a cgroup ("merge clusters") - * - * - They can remove a cluster from a cgroup ("break clusters"). - * - * After clustering, we also do some routine cleanup. Faces belonging to files - * that have been deleted (including those in Trash) should be pruned off. - * - * We should not make strict assumptions about the clusters we get from remote. - * In particular, the same face ID can be in different clusters. In such cases - * we should assign it arbitrarily assign it to the last cluster we find it in. - * Such leeway is intentionally provided to allow clients some slack in how they - * implement the sync without needing to make an blocking API request for every - * user interaction. */ export const clusterFaces = ( faceIndexes: FaceIndex[], diff --git a/web/packages/new/photos/services/ml/db.ts b/web/packages/new/photos/services/ml/db.ts index 5f57ea30e1..259d0ab5bc 100644 --- a/web/packages/new/photos/services/ml/db.ts +++ b/web/packages/new/photos/services/ml/db.ts @@ -2,8 +2,9 @@ import { removeKV } from "@/base/kv"; import log from "@/base/log"; import localForage from "@ente/shared/storage/localForage"; import { deleteDB, openDB, type DBSchema } from "idb"; +import type { CGroup } from "./cgroups"; import type { LocalCLIPIndex } from "./clip"; -import type { CGroup, FaceCluster } from "./cluster"; +import type { FaceCluster } from "./cluster"; import type { LocalFaceIndex } from "./face"; /** diff --git a/web/packages/new/photos/services/user-entity.ts b/web/packages/new/photos/services/user-entity.ts index 121171d214..ab76dce517 100644 --- a/web/packages/new/photos/services/user-entity.ts +++ b/web/packages/new/photos/services/user-entity.ts @@ -12,7 +12,7 @@ import { ensure } from "@/utils/ensure"; import { nullToUndefined } from "@/utils/transform"; import { z } from "zod"; import { gunzip } from "./gzip"; -import type { CGroup } from "./ml/cluster"; +import type { CGroup } from "./ml/cgroups"; import { applyCGroupDiff } from "./ml/db"; /** @@ -335,7 +335,7 @@ const saveLatestUpdatedAt = (type: EntityType, value: number) => * * This diff is then applied to the data we have persisted locally. */ -export const syncCGroups = async () => { +export const syncCGroupsWithRemote = async () => { const type: EntityType = "cgroup"; const entityKeyB64 = await getOrCreateEntityKeyB64(type); From 9af44e15b4c90a77d0ace825fad66f0770789f3b Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 4 Sep 2024 20:41:03 +0530 Subject: [PATCH 02/42] Keygen --- web/packages/base/crypto/index.ts | 10 ++++- web/packages/base/crypto/libsodium.ts | 13 +++++- .../new/photos/services/ml/cgroups.ts | 1 + .../new/photos/services/user-entity.ts | 41 +++++++++++-------- 4 files changed, 44 insertions(+), 21 deletions(-) diff --git a/web/packages/base/crypto/index.ts b/web/packages/base/crypto/index.ts index c2cf7bc005..eceea2408d 100644 --- a/web/packages/base/crypto/index.ts +++ b/web/packages/base/crypto/index.ts @@ -89,10 +89,16 @@ const assertInWorker = (x: T): T => { }; /** - * Generate a new randomly generated 256-bit key suitable for use with the *Box + * Return a new randomly generated 256-bit key suitable for use with the *Box * encryption functions. */ -export const generateBoxKey = libsodium.generateBoxKey; +export const generateNewBoxKey = libsodium.generateNewBoxKey; + +/** + * Return a new randomly generated 256-bit key suitable for use with the *Blob + * or *Stream encryption functions. + */ +export const generateNewBlobOrStreamKey = libsodium.generateNewBlobOrStreamKey; /** * Encrypt the given data, returning a box containing the encrypted data and a diff --git a/web/packages/base/crypto/libsodium.ts b/web/packages/base/crypto/libsodium.ts index cd91633c17..25ff0057e9 100644 --- a/web/packages/base/crypto/libsodium.ts +++ b/web/packages/base/crypto/libsodium.ts @@ -134,11 +134,22 @@ const bytes = async (bob: BytesOrB64) => * This returns a new randomly generated 256-bit key suitable for being used * with libsodium's secretbox APIs. */ -export const generateBoxKey = async () => { +export const generateNewBoxKey = async () => { await sodium.ready; return toB64(sodium.crypto_secretbox_keygen()); }; +/** + * Generate a key for use with the *Blob or *Stream encryption functions. + * + * This returns a new randomly generated 256-bit key suitable for being used + * with libsodium's secretstream APIs. + */ +export const generateNewBlobOrStreamKey = async () => { + await sodium.ready; + return toB64(sodium.crypto_secretstream_xchacha20poly1305_keygen()); +}; + /** * Encrypt the given data using libsodium's secretbox APIs, using a randomly * generated nonce. diff --git a/web/packages/new/photos/services/ml/cgroups.ts b/web/packages/new/photos/services/ml/cgroups.ts index 42274a7e37..49388eb990 100644 --- a/web/packages/new/photos/services/ml/cgroups.ts +++ b/web/packages/new/photos/services/ml/cgroups.ts @@ -118,6 +118,7 @@ export const syncCGroups = () => { // // The user can see both the cgroups and clusters in the UI, but only the // cgroups are synced. + // const syncCGroupsWithRemote() /* * After clustering, we also do some routine cleanup. Faces belonging to files * that have been deleted (including those in Trash) should be pruned off. diff --git a/web/packages/new/photos/services/user-entity.ts b/web/packages/new/photos/services/user-entity.ts index ab76dce517..927ed08168 100644 --- a/web/packages/new/photos/services/user-entity.ts +++ b/web/packages/new/photos/services/user-entity.ts @@ -2,7 +2,7 @@ import { decryptBlob, decryptBoxB64, encryptBoxB64, - generateBoxKey, + generateNewBlobOrStreamKey, } from "@/base/crypto"; import { authenticatedRequestHeaders, ensureOk, HTTPError } from "@/base/http"; import { getKV, getKVN, setKV } from "@/base/kv"; @@ -201,16 +201,15 @@ const getOrCreateEntityKeyB64 = async (type: EntityType) => { } // Nada. Create a new one, put it to remote, save it locally, and return. - // TODO-Cluster Keep this read only, only add the writeable bits after other - // stuff has been tested. - throw new Error("Not implemented"); - // const generatedKeyB64 = await worker.generateEncryptionKey(); - // const encryptedNewKey = await worker.encryptToB64( - // generatedKeyB64, - // encryptionKeyB64, - // ); - // await postUserEntityKey(type, newKey); - // return decrypt(newKey); + + // As a sanity check, genarate the key but immediately encrypt it as if it + // were fetched from remote and then try to decrypt it before doing anything + // with it. + const generated = await generateNewEncryptedEntityKey(); + const result = decryptEntityKey(generated); + await postUserEntityKey(type, generated); + await saveRemoteUserEntityKey(type, generated); + return result; }; const entityKeyKey = (type: EntityType) => `entityKey/${type}`; @@ -235,12 +234,17 @@ const saveRemoteUserEntityKey = ( ) => setKV(entityKeyKey(type), JSON.stringify(entityKey)); /** - * Generate a new entity key and return it after encrypting it using the user's - * master key. + * Generate a new entity key and return it in the shape of an + * {@link RemoteUserEntityKey} after encrypting it using the user's master key. */ -// TODO: Temporary export to silence lint -export const generateEncryptedEntityKey = async () => - encryptBoxB64(await generateBoxKey(), await masterKeyFromSession()); +const generateNewEncryptedEntityKey = async () => { + const { encryptedData, nonce } = await encryptBoxB64( + await generateNewBlobOrStreamKey(), + await masterKeyFromSession(), + ); + // Remote calls it the header, but it really is the nonce. + return { encryptedKey: encryptedData, header: nonce }; +}; /** * Decrypt an encrypted entity key using the user's master key. @@ -283,7 +287,9 @@ const getUserEntityKey = async ( }; const RemoteUserEntityKey = z.object({ + /** Base64 encoded entity key, encrypted with the user's master key. */ encryptedKey: z.string(), + /** Base64 encoded nonce used during encryption of this entity key. */ header: z.string(), }); @@ -294,8 +300,7 @@ type RemoteUserEntityKey = z.infer; * * See: [Note: User entity keys] */ -// TODO-Cluster remove export -export const postUserEntityKey = async ( +const postUserEntityKey = async ( type: EntityType, entityKey: RemoteUserEntityKey, ) => { From eb91b6ea6d287c3077c9db3785d0e420b669b6ed Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 5 Sep 2024 13:28:19 +0530 Subject: [PATCH 03/42] Doc --- web/apps/photos/src/services/sync.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/web/apps/photos/src/services/sync.ts b/web/apps/photos/src/services/sync.ts index c96e239526..ce6b03cbf3 100644 --- a/web/apps/photos/src/services/sync.ts +++ b/web/apps/photos/src/services/sync.ts @@ -16,9 +16,14 @@ export const triggerPreFileInfoSync = () => { }; /** - * Perform a soft "refresh" by making various API calls to fetch state from - * remote, using it to update our local state, and triggering periodic jobs that - * depend on the local state. + * Sync our local state with remote on page load for web and focus for desktop. + * + * This function makes various API calls to fetch state from remote, using it to + * update our local state, and triggering periodic jobs that depend on the local + * state. + * + * This runs on initial page load (on both web and desktop). In addition for + * desktop, it also runs each time the desktop app gains focus. * * TODO: This is called after we've synced the local files DBs with remote. That * code belongs here, but currently that state is persisted in the top level From 98979a2271b6ed095b1a59a192a2b256effb6399 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 5 Sep 2024 13:35:26 +0530 Subject: [PATCH 04/42] New path --- web/apps/photos/src/services/sync.ts | 8 ++++++-- web/packages/new/photos/services/user-entity.ts | 13 +++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/web/apps/photos/src/services/sync.ts b/web/apps/photos/src/services/sync.ts index ce6b03cbf3..db6946c612 100644 --- a/web/apps/photos/src/services/sync.ts +++ b/web/apps/photos/src/services/sync.ts @@ -4,6 +4,7 @@ import { triggerMLStatusSync, triggerMLSync, } from "@/new/photos/services/ml"; +import { syncUserEntities } from "@/new/photos/services/user-entity"; import { syncEntities } from "services/entityService"; import { syncMapEnabled } from "services/userService"; @@ -35,7 +36,10 @@ export const triggerPreFileInfoSync = () => { * before doing the file sync and thus should run immediately after login. */ export const sync = async () => { - await syncEntities(); - await syncMapEnabled(); + await Promise.allSettled([ + syncEntities(), + syncUserEntities(), + syncMapEnabled(), + ]); if (isMLSupported) triggerMLSync(); }; diff --git a/web/packages/new/photos/services/user-entity.ts b/web/packages/new/photos/services/user-entity.ts index 927ed08168..723c5ed7b4 100644 --- a/web/packages/new/photos/services/user-entity.ts +++ b/web/packages/new/photos/services/user-entity.ts @@ -9,6 +9,7 @@ import { getKV, getKVN, setKV } from "@/base/kv"; import { apiURL } from "@/base/origins"; import { masterKeyFromSession } from "@/base/session-store"; import { ensure } from "@/utils/ensure"; +import { wait } from "@/utils/promise"; import { nullToUndefined } from "@/utils/transform"; import { z } from "zod"; import { gunzip } from "./gzip"; @@ -30,6 +31,18 @@ export type EntityType = */ "cgroup"; +/** + * Sync our local state with the user entities present on remote. + * + * This function fetches all the user entity types that we are interested in for + * the photos app – location tags and cgroups – from remote and updates our + * local database. It uses local state to remember the last time it synced, so + * each subsequent sync is a lightweight diff. + */ +export const syncUserEntities = async () => { + return wait(0); +}; + /** * The maximum number of items to fetch in a single diff * From 785e96036abaf745e2c080d25374c2b416b3c2a6 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 5 Sep 2024 13:45:06 +0530 Subject: [PATCH 05/42] Add cleanup code --- .../new/photos/services/user-entity.ts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/web/packages/new/photos/services/user-entity.ts b/web/packages/new/photos/services/user-entity.ts index 723c5ed7b4..3039229e73 100644 --- a/web/packages/new/photos/services/user-entity.ts +++ b/web/packages/new/photos/services/user-entity.ts @@ -11,6 +11,7 @@ import { masterKeyFromSession } from "@/base/session-store"; import { ensure } from "@/utils/ensure"; import { wait } from "@/utils/promise"; import { nullToUndefined } from "@/utils/transform"; +import localForage from "@ente/shared/storage/localForage"; import { z } from "zod"; import { gunzip } from "./gzip"; import type { CGroup } from "./ml/cgroups"; @@ -40,9 +41,27 @@ export type EntityType = * each subsequent sync is a lightweight diff. */ export const syncUserEntities = async () => { + // TODO-cgroup: Call me + await Promise.allSettled([ + // removeLegacyDBState(), + ]); + return wait(0); }; +// TODO-cgroup: Call me +export const removeLegacyDBState = async () => { + // Older versions of the code kept the diff related state in a different + // place. These entries are not needed anymore. + // + // This code was added Sep 2024 and can be removed soon after a few builds + // have gone out (tag: Migration). + await Promise.allSettled([ + localForage.removeItem("location_tags"), + localForage.removeItem("location_tags_key"), + localForage.removeItem("location_tags_key"), + ]); +}; /** * The maximum number of items to fetch in a single diff * From 57e7eb9e05dbe8b9813b06067fd031502bb4b7a7 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 5 Sep 2024 14:11:31 +0530 Subject: [PATCH 06/42] Keep them separate --- web/apps/photos/src/services/sync.ts | 4 +- .../new/photos/services/user-entity.ts | 193 +++++++++++------- 2 files changed, 116 insertions(+), 81 deletions(-) diff --git a/web/apps/photos/src/services/sync.ts b/web/apps/photos/src/services/sync.ts index db6946c612..c42f55d89f 100644 --- a/web/apps/photos/src/services/sync.ts +++ b/web/apps/photos/src/services/sync.ts @@ -4,7 +4,7 @@ import { triggerMLStatusSync, triggerMLSync, } from "@/new/photos/services/ml"; -import { syncUserEntities } from "@/new/photos/services/user-entity"; +import { syncLocationTags } from "@/new/photos/services/user-entity"; import { syncEntities } from "services/entityService"; import { syncMapEnabled } from "services/userService"; @@ -38,7 +38,7 @@ export const triggerPreFileInfoSync = () => { export const sync = async () => { await Promise.allSettled([ syncEntities(), - syncUserEntities(), + syncLocationTags(), syncMapEnabled(), ]); if (isMLSupported) triggerMLSync(); diff --git a/web/packages/new/photos/services/user-entity.ts b/web/packages/new/photos/services/user-entity.ts index 3039229e73..8823c5d8f5 100644 --- a/web/packages/new/photos/services/user-entity.ts +++ b/web/packages/new/photos/services/user-entity.ts @@ -9,7 +9,6 @@ import { getKV, getKVN, setKV } from "@/base/kv"; import { apiURL } from "@/base/origins"; import { masterKeyFromSession } from "@/base/session-store"; import { ensure } from "@/utils/ensure"; -import { wait } from "@/utils/promise"; import { nullToUndefined } from "@/utils/transform"; import localForage from "@ente/shared/storage/localForage"; import { z } from "zod"; @@ -33,20 +32,38 @@ export type EntityType = "cgroup"; /** - * Sync our local state with the user entities present on remote. + * Sync our local location tags with those on remote. * - * This function fetches all the user entity types that we are interested in for - * the photos app – location tags and cgroups – from remote and updates our - * local database. It uses local state to remember the last time it synced, so - * each subsequent sync is a lightweight diff. + * This function fetches all the location tag user entities from remote and + * updates our local database. It uses local state to remember the last time it + * synced, so each subsequent sync is a lightweight diff. */ -export const syncUserEntities = async () => { - // TODO-cgroup: Call me - await Promise.allSettled([ - // removeLegacyDBState(), - ]); +export const syncLocationTags = async () => { + // TODO-cgroup: Implement me + const parse = async (id: string, data: Uint8Array): Promise => { + const rp = RemoteCGroup.parse(JSON.parse(await gunzip(data))); + return { + id, + name: rp.name, + clusterIDs: rp.assigned.map(({ id }) => id), + isHidden: rp.isHidden, + avatarFaceID: rp.avatarFaceID, + displayFaceID: undefined, + }; + }; - return wait(0); + const processBatch = async (entities: UserEntityChange[]) => + await applyCGroupDiff( + await Promise.all( + entities.map(async ({ id, data }) => + data ? await parse(id, data) : id, + ), + ), + ); + + // TODO-cgroup: Call me + // await removeLegacyDBState(); + return syncUserEntity("cgroup", processBatch); }; // TODO-cgroup: Call me @@ -59,9 +76,62 @@ export const removeLegacyDBState = async () => { await Promise.allSettled([ localForage.removeItem("location_tags"), localForage.removeItem("location_tags_key"), - localForage.removeItem("location_tags_key"), + localForage.removeItem("location_tags_time"), ]); }; + +/** + * Sync the {@link CGroup} entities that we have locally with remote. + * + * This fetches all the user entities corresponding to the "cgroup" entity type + * from remote that have been created, updated or deleted since the last time we + * checked. + * + * This diff is then applied to the data we have persisted locally. + */ +export const syncCGroups = () => { + const parse = async (id: string, data: Uint8Array): Promise => { + const rp = RemoteCGroup.parse(JSON.parse(await gunzip(data))); + return { + id, + name: rp.name, + clusterIDs: rp.assigned.map(({ id }) => id), + isHidden: rp.isHidden, + avatarFaceID: rp.avatarFaceID, + displayFaceID: undefined, + }; + }; + + const processBatch = async (entities: UserEntityChange[]) => + await applyCGroupDiff( + await Promise.all( + entities.map(async ({ id, data }) => + data ? await parse(id, data) : id, + ), + ), + ); + + return syncUserEntity("cgroup", processBatch); +}; + +/** Zod schema for the {@link RemoteCGroup} type. */ +const RemoteCGroup = z.object({ + name: z.string().nullish().transform(nullToUndefined), + assigned: z.array( + z.object({ + id: z.string(), + faces: z.string().array(), + }), + ), + isHidden: z.boolean(), + avatarFaceID: z.string().nullish().transform(nullToUndefined), +}); + +/** + * Contents of a "cgroup" user entity, as synced via remote. + */ +type RemoteCGroup = z.infer; + /** * The maximum number of items to fetch in a single diff * @@ -114,6 +184,37 @@ interface UserEntityChange { updatedAt: number; } +/** + * Sync of the given {@link type} entities that we have locally with remote. + * + * This fetches all the user entities of {@link type} from remote that have been + * created, updated or deleted since the last time we checked. + * + * For each diff response, the {@link processBatch} is invoked to give a chance + * to caller to apply the updates to the data we have persisted locally. + */ +const syncUserEntity = async ( + type: EntityType, + processBatch: (entities: UserEntityChange[]) => Promise, +) => { + const entityKeyB64 = await getOrCreateEntityKeyB64(type); + + let sinceTime = (await savedLatestUpdatedAt(type)) ?? 0; + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition, no-constant-condition + while (true) { + const entities = await userEntityDiff(type, sinceTime, entityKeyB64); + if (entities.length == 0) break; + + await processBatch(entities); + + sinceTime = entities.reduce( + (max, entity) => Math.max(max, entity.updatedAt), + sinceTime, + ); + await saveLatestUpdatedAt(type, sinceTime); + } +}; + /** * Zod schema for a item in the user entity diff. */ @@ -362,69 +463,3 @@ const savedLatestUpdatedAt = (type: EntityType) => */ const saveLatestUpdatedAt = (type: EntityType, value: number) => setKV(latestUpdatedAtKey(type), value); - -/** - * Sync the {@link CGroup} entities that we have locally with remote. - * - * This fetches all the user entities corresponding to the "cgroup" entity type - * from remote that have been created, updated or deleted since the last time we - * checked. - * - * This diff is then applied to the data we have persisted locally. - */ -export const syncCGroupsWithRemote = async () => { - const type: EntityType = "cgroup"; - - const entityKeyB64 = await getOrCreateEntityKeyB64(type); - - const parse = async (id: string, data: Uint8Array): Promise => { - const rp = RemoteCGroup.parse(JSON.parse(await gunzip(data))); - return { - id, - name: rp.name, - clusterIDs: rp.assigned.map(({ id }) => id), - isHidden: rp.isHidden, - avatarFaceID: rp.avatarFaceID, - displayFaceID: undefined, - }; - }; - - let sinceTime = (await savedLatestUpdatedAt(type)) ?? 0; - // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition, no-constant-condition - while (true) { - const entities = await userEntityDiff(type, sinceTime, entityKeyB64); - if (entities.length == 0) break; - - await applyCGroupDiff( - await Promise.all( - entities.map(async ({ id, data }) => - data ? await parse(id, data) : id, - ), - ), - ); - - sinceTime = entities.reduce( - (max, entity) => Math.max(max, entity.updatedAt), - sinceTime, - ); - await saveLatestUpdatedAt(type, sinceTime); - } -}; - -/** Zod schema for the {@link RemoteCGroup} type. */ -const RemoteCGroup = z.object({ - name: z.string().nullish().transform(nullToUndefined), - assigned: z.array( - z.object({ - id: z.string(), - faces: z.string().array(), - }), - ), - isHidden: z.boolean(), - avatarFaceID: z.string().nullish().transform(nullToUndefined), -}); - -/** - * Contents of a "cgroup" user entity, as synced via remote. - */ -type RemoteCGroup = z.infer; From a9be915f87680d5774041b2c6371ce2d8078a670 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 5 Sep 2024 15:20:13 +0530 Subject: [PATCH 07/42] Remote Loc 1 --- .../new/photos/services/user-entity.ts | 60 +++++++++++-------- 1 file changed, 35 insertions(+), 25 deletions(-) diff --git a/web/packages/new/photos/services/user-entity.ts b/web/packages/new/photos/services/user-entity.ts index 8823c5d8f5..1493d4efaf 100644 --- a/web/packages/new/photos/services/user-entity.ts +++ b/web/packages/new/photos/services/user-entity.ts @@ -9,6 +9,7 @@ import { getKV, getKVN, setKV } from "@/base/kv"; import { apiURL } from "@/base/origins"; import { masterKeyFromSession } from "@/base/session-store"; import { ensure } from "@/utils/ensure"; +import { wait } from "@/utils/promise"; import { nullToUndefined } from "@/utils/transform"; import localForage from "@ente/shared/storage/localForage"; import { z } from "zod"; @@ -23,13 +24,23 @@ import { applyCGroupDiff } from "./ml/db"; * e.g. location tags, cluster groups. */ export type EntityType = + /** + * A location tag. + * + * The entity data is base64(encrypt(json)) + */ + | "location" /** * A cluster group. * - * Format: An encrypted string containing a gzipped JSON string representing - * the cgroup data. + * The entity data is base64(encrypt(gzip(json))) */ - "cgroup"; + | "cgroup"; + +interface LocationTag { + id: string; + name: string; +} /** * Sync our local location tags with those on remote. @@ -39,31 +50,25 @@ export type EntityType = * synced, so each subsequent sync is a lightweight diff. */ export const syncLocationTags = async () => { - // TODO-cgroup: Implement me - const parse = async (id: string, data: Uint8Array): Promise => { - const rp = RemoteCGroup.parse(JSON.parse(await gunzip(data))); + const decoder = new TextDecoder(); + const parse = (id: string, data: Uint8Array): LocationTag => { + const rl = RemoteLocation.parse(JSON.parse(decoder.decode(data))); return { id, - name: rp.name, - clusterIDs: rp.assigned.map(({ id }) => id), - isHidden: rp.isHidden, - avatarFaceID: rp.avatarFaceID, - displayFaceID: undefined, + name: rl.name, }; }; - const processBatch = async (entities: UserEntityChange[]) => - await applyCGroupDiff( - await Promise.all( - entities.map(async ({ id, data }) => - data ? await parse(id, data) : id, - ), - ), + const processBatch = async (entities: UserEntityChange[]) => { + console.log( + entities.map(({ id, data }) => (data ? parse(id, data) : id)), ); + await wait(0); + }; // TODO-cgroup: Call me // await removeLegacyDBState(); - return syncUserEntity("cgroup", processBatch); + return syncUserEntity("location", processBatch); }; // TODO-cgroup: Call me @@ -80,6 +85,17 @@ export const removeLegacyDBState = async () => { ]); }; +const RemoteLocation = z.object({ + name: z.string(), + radius: z.number(), + aSquare: z.number(), + bSquare: z.number(), + centerPoint: z.object({ + latitude: z.number().nullish().transform(nullToUndefined), + longitude: z.number().nullish().transform(nullToUndefined), + }), +}); + /** * Sync the {@link CGroup} entities that we have locally with remote. * @@ -114,7 +130,6 @@ export const syncCGroups = () => { return syncUserEntity("cgroup", processBatch); }; -/** Zod schema for the {@link RemoteCGroup} type. */ const RemoteCGroup = z.object({ name: z.string().nullish().transform(nullToUndefined), assigned: z.array( @@ -127,11 +142,6 @@ const RemoteCGroup = z.object({ avatarFaceID: z.string().nullish().transform(nullToUndefined), }); -/** - * Contents of a "cgroup" user entity, as synced via remote. - */ -type RemoteCGroup = z.infer; - /** * The maximum number of items to fetch in a single diff * From c90315679f6602eed5e5f49ee08159a2c39b1db4 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 5 Sep 2024 15:56:41 +0530 Subject: [PATCH 08/42] Migrator --- .../new/photos/services/migrations.ts | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 web/packages/new/photos/services/migrations.ts diff --git a/web/packages/new/photos/services/migrations.ts b/web/packages/new/photos/services/migrations.ts new file mode 100644 index 0000000000..37bd84959f --- /dev/null +++ b/web/packages/new/photos/services/migrations.ts @@ -0,0 +1,63 @@ +import { isDesktop } from "@/base/app"; +import { getKVN, removeKV, setKV } from "@/base/kv"; +import log from "@/base/log"; +import localForage from "@ente/shared/storage/localForage"; +import { deleteDB } from "idb"; + +/** + * App specific migrations. + * + * The app stores data in multiple places: local storage, IndexedDB, OPFS, and + * not all of these support DB migrations. And even when they do, those are + * rather heavy weight and complicated (e.g. IndexedDB). + * + * Further, there are various app level migrations, e.g. resetting the diff + * fetch times, that don't correspond to DB migrations, these are just changes + * we need to make to our locally persisted values and not the schemas + * themselves. + * + * Thus we introduce the concept of app level migrations. This is some code + * which runs early in the page load, and runs arbitrary blocks of code until it + * reaches the last migration number. + * + * We can put all sorts of changes here: cleanup of legacy keys, re-triggers for + * various fetches etc. + * + * This code usually runs fairly early on page load, but if you need specific + * guarantees or have dependencies in the order of operations (beyond what is + * captured by the sequential flow here), then this might not be appropriate. + */ +export const runMigrations = async () => { + const m = (await getKVN("migrationLevel")) ?? 0; + const latest = 1; + if (m < latest) { + log.info(`Running migrations ${m} => ${latest}`); + if (m < 1 && isDesktop) await m0(); + await setKV("migrationLevel", latest); + } +}; + +// Some of these (indicated by "Prunable") can be no-oped in the future when +// almost all clients would've migrated over. + +// Last used: Aug 2024. Prunable. +const m0 = () => + Promise.allSettled([ + // Delete the legacy face DB v1. + deleteDB("mldata"), + + // Delete the legacy CLIP (mostly) related keys from LocalForage. + localForage.removeItem("embeddings"), + localForage.removeItem("embedding_sync_time"), + localForage.removeItem("embeddings_v2"), + localForage.removeItem("file_embeddings"), + localForage.removeItem("onnx-clip-embedding_sync_time"), + localForage.removeItem("file-ml-clip-face-embedding_sync_time"), + + // Delete keys for the legacy diff based sync. + removeKV("embeddingSyncTime:onnx-clip"), + removeKV("embeddingSyncTime:file-ml-clip-face"), + + // Delete the legacy face DB v2. + deleteDB("face"), + ]); From 5fd0b46756439ce96c4bb06a882a0069a18fba7c Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 5 Sep 2024 15:58:15 +0530 Subject: [PATCH 09/42] Remove legacy apiOrigin migration > "Note that the legacy value was never in production builds, only nightlies, so this code can be removed soon" --- web/packages/base/origins.ts | 22 +++++-------------- .../new/photos/components/DevSettings.tsx | 18 +-------------- 2 files changed, 6 insertions(+), 34 deletions(-) diff --git a/web/packages/base/origins.ts b/web/packages/base/origins.ts index 8c39396fb2..d1474014a3 100644 --- a/web/packages/base/origins.ts +++ b/web/packages/base/origins.ts @@ -1,5 +1,4 @@ -import { getKV, setKV } from "@/base/kv"; -import { inWorker } from "./env"; +import { getKV } from "@/base/kv"; /** * Return the origin (scheme, host, port triple) that should be used for making @@ -35,21 +34,10 @@ export const apiURL = async (path: string) => (await apiOrigin()) + path; * * Otherwise return undefined. */ -export const customAPIOrigin = async () => { - let origin = await getKV("apiOrigin"); - if (!origin && !inWorker()) { - // TODO: Migration of apiOrigin from local storage to indexed DB. Added - // 27 June 2024, 1.7.2-rc. Remove me after a bit (tag: Migration). - const legacyOrigin = localStorage.getItem("apiOrigin"); - if (legacyOrigin !== null) { - origin = legacyOrigin; - if (origin) await setKV("apiOrigin", origin); - localStorage.removeItem("apiOrigin"); - } - } - - return origin ?? process.env.NEXT_PUBLIC_ENTE_ENDPOINT ?? undefined; -}; +export const customAPIOrigin = async () => + (await getKV("apiOrigin")) ?? + process.env.NEXT_PUBLIC_ENTE_ENDPOINT ?? + undefined; /** * A convenience wrapper over {@link customAPIOrigin} that returns the only the diff --git a/web/packages/new/photos/components/DevSettings.tsx b/web/packages/new/photos/components/DevSettings.tsx index 2e3b1b3978..33da3a7373 100644 --- a/web/packages/new/photos/components/DevSettings.tsx +++ b/web/packages/new/photos/components/DevSettings.tsx @@ -69,17 +69,7 @@ const Contents: React.FC = (props) => { >(); useEffect( - () => - void getKV("apiOrigin").then((o) => - setInitialAPIOrigin( - // Migrate apiOrigin from local storage to indexed DB. - // - // This code was added 27 June 2024. Note that the legacy - // value was never in production builds, only nightlies, so - // this code can be removed soon (tag: Migration). - o ?? localStorage.getItem("apiOrigin") ?? "", - ), - ), + () => void getKV("apiOrigin").then((o) => setInitialAPIOrigin(o ?? "")), [], ); @@ -219,12 +209,6 @@ const Form: React.FC = ({ initialAPIOrigin, onClose }) => { const updateAPIOrigin = async (origin: string) => { if (!origin) { await removeKV("apiOrigin"); - // Migrate apiOrigin from local storage to indexed DB. - // - // This code was added 27 June 2024. Note that the legacy value was - // never in production builds, only nightlies, so this code can be - // removed at some point soon (tag: Migration). - localStorage.removeItem("apiOrigin"); return; } From 13199bb3f74b922678d18def22bae9ac7a2a8faf Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 5 Sep 2024 16:06:54 +0530 Subject: [PATCH 10/42] Use --- web/apps/photos/src/pages/_app.tsx | 2 + .../new/photos/services/migrations.ts | 5 ++- web/packages/new/photos/services/ml/db.ts | 42 ------------------- web/packages/new/photos/services/ml/index.ts | 5 --- 4 files changed, 6 insertions(+), 48 deletions(-) diff --git a/web/apps/photos/src/pages/_app.tsx b/web/apps/photos/src/pages/_app.tsx index 15d2c58eee..53acd2e386 100644 --- a/web/apps/photos/src/pages/_app.tsx +++ b/web/apps/photos/src/pages/_app.tsx @@ -4,6 +4,7 @@ import { CustomHead } from "@/base/components/Head"; import { AppNavbar } from "@/base/components/Navbar"; import { setupI18n } from "@/base/i18n"; import log from "@/base/log"; +import { runMigrations } from "@/new/photos/services/migrations"; import { logStartupBanner, logUnhandledErrorsAndRejections, @@ -141,6 +142,7 @@ export default function App({ Component, pageProps }: AppProps) { logStartupBanner(user?.id); HTTPService.setHeaders({ "X-Client-Package": clientPackageName }); logUnhandledErrorsAndRejections(true); + void runMigrations(); return () => logUnhandledErrorsAndRejections(false); }, []); diff --git a/web/packages/new/photos/services/migrations.ts b/web/packages/new/photos/services/migrations.ts index 37bd84959f..639b606f46 100644 --- a/web/packages/new/photos/services/migrations.ts +++ b/web/packages/new/photos/services/migrations.ts @@ -60,4 +60,7 @@ const m0 = () => // Delete the legacy face DB v2. deleteDB("face"), - ]); + ]).then(() => { + // Delete legacy ML keys. + localStorage.removeItem("faceIndexingEnabled"); + }); diff --git a/web/packages/new/photos/services/ml/db.ts b/web/packages/new/photos/services/ml/db.ts index 259d0ab5bc..b894505828 100644 --- a/web/packages/new/photos/services/ml/db.ts +++ b/web/packages/new/photos/services/ml/db.ts @@ -1,6 +1,4 @@ -import { removeKV } from "@/base/kv"; import log from "@/base/log"; -import localForage from "@ente/shared/storage/localForage"; import { deleteDB, openDB, type DBSchema } from "idb"; import type { CGroup } from "./cgroups"; import type { LocalCLIPIndex } from "./clip"; @@ -143,44 +141,6 @@ const openMLDB = async () => { return db; }; -const deleteLegacyDB = () => { - // Delete the legacy face DB v1. - // - // This code was added June 2024 (v1.7.1-rc) and can be removed at some - // point when most clients have migrated (tag: Migration). - void deleteDB("mldata"); - - // Delete the legacy CLIP (mostly) related keys from LocalForage. - // - // This code was added July 2024 (v1.7.2-rc) and can be removed at some - // point when most clients have migrated (tag: Migration). - void Promise.all([ - localForage.removeItem("embeddings"), - localForage.removeItem("embedding_sync_time"), - localForage.removeItem("embeddings_v2"), - localForage.removeItem("file_embeddings"), - localForage.removeItem("onnx-clip-embedding_sync_time"), - localForage.removeItem("file-ml-clip-face-embedding_sync_time"), - ]); - - // Delete keys for the legacy diff based sync. - // - // This code was added July 2024 (v1.7.3-beta). These keys were never - // enabled outside of the nightly builds, so this cleanup is not a hard - // need. Either ways, it can be removed at some point when most clients have - // migrated (tag: Migration). - void Promise.all([ - removeKV("embeddingSyncTime:onnx-clip"), - removeKV("embeddingSyncTime:file-ml-clip-face"), - ]); - - // Delete the legacy face DB v2. - // - // This code was added Aug 2024 (v1.7.3-beta) and can be removed at some - // point when most clients have migrated (tag: Migration). - void deleteDB("face"); -}; - /** * @returns a lazily created, cached connection to the ML DB. */ @@ -192,8 +152,6 @@ const mlDB = () => (_mlDB ??= openMLDB()); * This is meant to be called during logout on the main thread. */ export const clearMLDB = async () => { - deleteLegacyDB(); - try { if (_mlDB) (await _mlDB).close(); } catch (e) { diff --git a/web/packages/new/photos/services/ml/index.ts b/web/packages/new/photos/services/ml/index.ts index b3ce72ae0d..1d43c1cdc6 100644 --- a/web/packages/new/photos/services/ml/index.ts +++ b/web/packages/new/photos/services/ml/index.ts @@ -229,11 +229,6 @@ const mlLocalKey = "mlEnabled"; * that is synced with remote. */ const isMLEnabledLocal = () => { - // Delete legacy ML keys. - // - // This code was added August 2024 (v1.7.3-beta) and can be removed at some - // point when most clients have migrated (tag: Migration). - localStorage.removeItem("faceIndexingEnabled"); return localStorage.getItem(mlLocalKey) == "1"; }; From c67a6b0c9e54815c22b9fb146c8aea9305e4d5bc Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 5 Sep 2024 16:25:53 +0530 Subject: [PATCH 11/42] Clean --- web/packages/new/photos/services/ml/db.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/web/packages/new/photos/services/ml/db.ts b/web/packages/new/photos/services/ml/db.ts index b894505828..86cc0afd03 100644 --- a/web/packages/new/photos/services/ml/db.ts +++ b/web/packages/new/photos/services/ml/db.ts @@ -106,8 +106,6 @@ interface FileStatus { let _mlDB: ReturnType | undefined; const openMLDB = async () => { - deleteLegacyDB(); - const db = await openDB("ml", 1, { upgrade(db, oldVersion, newVersion) { log.info(`Upgrading ML DB ${oldVersion} => ${newVersion}`); From 6e8514e08c17a18086a4769d3586463a09fe069c Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 5 Sep 2024 16:11:47 +0530 Subject: [PATCH 12/42] Add sanity checker --- .../shared/storage/localStorage/index.ts | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/web/packages/shared/storage/localStorage/index.ts b/web/packages/shared/storage/localStorage/index.ts index df80b21330..4313c99b74 100644 --- a/web/packages/shared/storage/localStorage/index.ts +++ b/web/packages/shared/storage/localStorage/index.ts @@ -1,4 +1,4 @@ -import { removeKV, setKV } from "@/base/kv"; +import { getKV, removeKV, setKV } from "@/base/kv"; import log from "@/base/log"; export enum LS_KEYS { @@ -68,6 +68,15 @@ export const setLSUser = async (user: object) => { * inlined into `setLSUser` (tag: Migration). */ export const migrateKVToken = async (user: unknown) => { + // Throw an error if the data is in local storage but not in IndexedDB. This + // is a pre-cursor to inlining this code. + const hadMismatch = + user && + typeof user == "object" && + "token" in user && + typeof user.token == "string" && + !(await getKV("token")); + user && typeof user == "object" && "id" in user && @@ -81,4 +90,9 @@ export const migrateKVToken = async (user: unknown) => { typeof user.token == "string" ? await setKV("token", user.token) : await removeKV("token"); + + if (hadMismatch) + throw new Error( + "The user's token was present in local storage but not in IndexedDB", + ); }; From 8cfe36be688b304ebc1ad3b823d0e85e20e70037 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 5 Sep 2024 16:21:33 +0530 Subject: [PATCH 13/42] Remove legacy locale mapper --- web/packages/base/i18n.ts | 68 +------------------ .../shared/storage/localStorage/index.ts | 4 +- 2 files changed, 3 insertions(+), 69 deletions(-) diff --git a/web/packages/base/i18n.ts b/web/packages/base/i18n.ts index 11121b7a52..5067e5fc95 100644 --- a/web/packages/base/i18n.ts +++ b/web/packages/base/i18n.ts @@ -5,7 +5,6 @@ import { getUserLocales } from "get-user-locale"; import i18n from "i18next"; import resourcesToBackend from "i18next-resources-to-backend"; import { initReactI18next } from "react-i18next"; -import { object, string } from "yup"; /** * List of all {@link SupportedLocale}s. @@ -60,7 +59,7 @@ const defaultLocale: SupportedLocale = "en-US"; * produce a string like "July 19, 2024". */ export const setupI18n = async () => { - const localeString = savedLocaleStringMigratingIfNeeded(); + const localeString = localStorage.getItem("locale") ?? undefined; const locale = closestSupportedLocale(localeString); // https://www.i18next.com/overview/api @@ -136,71 +135,6 @@ export const setupI18n = async () => { }); }; -/** - * Read and return the locale (if any) that we'd previously saved in local - * storage. - * - * If it finds a locale stored in the old format, it also updates the saved - * value and returns it in the new format. - */ -const savedLocaleStringMigratingIfNeeded = (): SupportedLocale | undefined => { - const ls = localStorage.getItem("locale"); - - // An older version of our code had stored only the language code, not the - // full locale. Migrate these to the new locale format. Luckily, all such - // languages can be unambiguously mapped to locales in our current set. - // - // This migration is dated Feb 2024. And it can be removed after a few - // months, because by then either customers would've opened the app and - // their setting migrated to the new format, or the browser would've cleared - // the older local storage entry anyway (tag: Migration). - - if (!ls) { - // Nothing found - return undefined; - } - - if (includes(supportedLocales, ls)) { - // Already in the new format - return ls; - } - - let value: string | undefined; - try { - const oldFormatData = object({ value: string() }).json().cast(ls); - value = oldFormatData.value; - } catch (e) { - // Not a valid JSON, or not in the format we expected it. This shouldn't - // have happened, we're the only one setting it. - log.error("Failed to parse locale obtained from local storage", e); - // Also remove the old key, it is not parseable by us anymore. - localStorage.removeItem("locale"); - return undefined; - } - - const newValue = mapOldValue(value); - if (newValue) localStorage.setItem("locale", newValue); - - return newValue; -}; - -const mapOldValue = (value: string | undefined) => { - switch (value) { - case "en": - return "en-US"; - case "fr": - return "fr-FR"; - case "zh": - return "zh-CN"; - case "nl": - return "nl-NL"; - case "es": - return "es-ES"; - default: - return undefined; - } -}; - /** * Return the closest / best matching {@link SupportedLocale}. * diff --git a/web/packages/shared/storage/localStorage/index.ts b/web/packages/shared/storage/localStorage/index.ts index 4313c99b74..6330f21d9a 100644 --- a/web/packages/shared/storage/localStorage/index.ts +++ b/web/packages/shared/storage/localStorage/index.ts @@ -70,7 +70,7 @@ export const setLSUser = async (user: object) => { export const migrateKVToken = async (user: unknown) => { // Throw an error if the data is in local storage but not in IndexedDB. This // is a pre-cursor to inlining this code. - const hadMismatch = + const wasMissing = user && typeof user == "object" && "token" in user && @@ -91,7 +91,7 @@ export const migrateKVToken = async (user: unknown) => { ? await setKV("token", user.token) : await removeKV("token"); - if (hadMismatch) + if (wasMissing) throw new Error( "The user's token was present in local storage but not in IndexedDB", ); From 9476d2697261436d361b4cd8e18f25c406b9fb29 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 5 Sep 2024 16:27:38 +0530 Subject: [PATCH 14/42] Mention version --- web/packages/new/photos/services/migrations.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/packages/new/photos/services/migrations.ts b/web/packages/new/photos/services/migrations.ts index 639b606f46..6123f0c879 100644 --- a/web/packages/new/photos/services/migrations.ts +++ b/web/packages/new/photos/services/migrations.ts @@ -40,7 +40,7 @@ export const runMigrations = async () => { // Some of these (indicated by "Prunable") can be no-oped in the future when // almost all clients would've migrated over. -// Last used: Aug 2024. Prunable. +// Added: Aug 2024 (v1.7.3). Prunable. const m0 = () => Promise.allSettled([ // Delete the legacy face DB v1. From 18e3adde111725d7090a98888bf7e897d3d85b2b Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 5 Sep 2024 19:19:24 +0530 Subject: [PATCH 15/42] Refactor search code In prep for moving location tags handling to @/new --- .../src/components/Search/SearchBar/index.tsx | 2 +- .../SearchBar/searchInput/MenuWithPeople.tsx | 2 +- .../Search/SearchBar/searchInput/index.tsx | 20 +-- .../SearchBar/searchInput/optionWithInfo.tsx | 2 +- .../searchInput/valueContainerWithIcon.tsx | 5 +- .../components/Search/SearchResultInfo.tsx | 2 +- .../src/components/pages/gallery/Navbar.tsx | 2 +- web/apps/photos/src/pages/_app.tsx | 2 +- web/apps/photos/src/pages/gallery/index.tsx | 62 ++------ web/apps/photos/src/services/heic-convert.ts | 0 .../src/services/locationSearchService.ts | 54 +------ web/apps/photos/src/services/searchService.ts | 30 ++-- web/apps/photos/src/types/entity.ts | 15 +- web/apps/photos/src/types/search/index.ts | 65 -------- .../src/utils/comlink/ComlinkSearchWorker.ts | 32 ---- web/apps/photos/src/worker/search.worker.ts | 88 ----------- .../new/photos/services/search/index.ts | 39 ++++- .../new/photos/services/search/types.ts | 84 +++++++++- .../new/photos/services/search/worker.ts | 146 ++++++++++++++++++ web/packages/shared/file-metadata.ts | 2 +- 20 files changed, 317 insertions(+), 337 deletions(-) delete mode 100644 web/apps/photos/src/services/heic-convert.ts delete mode 100644 web/apps/photos/src/utils/comlink/ComlinkSearchWorker.ts delete mode 100644 web/apps/photos/src/worker/search.worker.ts create mode 100644 web/packages/new/photos/services/search/worker.ts diff --git a/web/apps/photos/src/components/Search/SearchBar/index.tsx b/web/apps/photos/src/components/Search/SearchBar/index.tsx index 8d7612d4b5..fa8929e4e6 100644 --- a/web/apps/photos/src/components/Search/SearchBar/index.tsx +++ b/web/apps/photos/src/components/Search/SearchBar/index.tsx @@ -1,8 +1,8 @@ import { Collection } from "types/collection"; import { SearchBarMobile } from "./searchBarMobile"; +import { UpdateSearch } from "@/new/photos/services/search/types"; import { EnteFile } from "@/new/photos/types/file"; -import { UpdateSearch } from "types/search"; import SearchInput from "./searchInput"; import { SearchBarWrapper } from "./styledComponents"; diff --git a/web/apps/photos/src/components/Search/SearchBar/searchInput/MenuWithPeople.tsx b/web/apps/photos/src/components/Search/SearchBar/searchInput/MenuWithPeople.tsx index 25aa944d8e..4d924e7c1b 100644 --- a/web/apps/photos/src/components/Search/SearchBar/searchInput/MenuWithPeople.tsx +++ b/web/apps/photos/src/components/Search/SearchBar/searchInput/MenuWithPeople.tsx @@ -1,10 +1,10 @@ import { PeopleList } from "@/new/photos/components/PeopleList"; import { isMLEnabled } from "@/new/photos/services/ml"; +import { Suggestion, SuggestionType } from "@/new/photos/services/search/types"; import { Row } from "@ente/shared/components/Container"; import { Box, styled } from "@mui/material"; import { t } from "i18next"; import { components } from "react-select"; -import { Suggestion, SuggestionType } from "types/search"; const { Menu } = components; diff --git a/web/apps/photos/src/components/Search/SearchBar/searchInput/index.tsx b/web/apps/photos/src/components/Search/SearchBar/searchInput/index.tsx index 18b7289915..0f85391ad0 100644 --- a/web/apps/photos/src/components/Search/SearchBar/searchInput/index.tsx +++ b/web/apps/photos/src/components/Search/SearchBar/searchInput/index.tsx @@ -1,9 +1,18 @@ import { FileType } from "@/media/file-type"; import { isMLEnabled } from "@/new/photos/services/ml"; import type { + City, + LocationTagData, SearchDateComponents, SearchPerson, } from "@/new/photos/services/search/types"; +import { + ClipSearchScores, + SearchOption, + SearchQuery, + SuggestionType, + UpdateSearch, +} from "@/new/photos/services/search/types"; import { EnteFile } from "@/new/photos/types/file"; import CloseIcon from "@mui/icons-material/Close"; import { IconButton } from "@mui/material"; @@ -15,20 +24,11 @@ import { useCallback, useContext, useEffect, useRef, useState } from "react"; import { components } from "react-select"; import AsyncSelect from "react-select/async"; import { InputActionMeta } from "react-select/src/types"; -import { City } from "services/locationSearchService"; import { getAutoCompleteSuggestions, getDefaultOptions, } from "services/searchService"; import { Collection } from "types/collection"; -import { LocationTagData } from "types/entity"; -import { - ClipSearchScores, - Search, - SearchOption, - SuggestionType, - UpdateSearch, -} from "types/search"; import { SelectStyles } from "../../../../styles/search"; import { SearchInputWrapper } from "../styledComponents"; import MenuWithPeople from "./MenuWithPeople"; @@ -116,7 +116,7 @@ export default function SearchInput(props: Iprops) { if (!selectedOption) { return; } - let search: Search; + let search: SearchQuery; switch (selectedOption.type) { case SuggestionType.DATE: search = { diff --git a/web/apps/photos/src/components/Search/SearchBar/searchInput/optionWithInfo.tsx b/web/apps/photos/src/components/Search/SearchBar/searchInput/optionWithInfo.tsx index 8e3fd7d842..62a8d72b4b 100644 --- a/web/apps/photos/src/components/Search/SearchBar/searchInput/optionWithInfo.tsx +++ b/web/apps/photos/src/components/Search/SearchBar/searchInput/optionWithInfo.tsx @@ -1,3 +1,4 @@ +import { SearchOption } from "@/new/photos/services/search/types"; import { FreeFlowText, SpaceBetweenFlex, @@ -6,7 +7,6 @@ import { Box, Divider, Stack, Typography } from "@mui/material"; import CollectionCard from "components/Collections/CollectionCard"; import { ResultPreviewTile } from "components/Collections/styledComponents"; import { t } from "i18next"; -import { SearchOption } from "types/search"; import { components } from "react-select"; diff --git a/web/apps/photos/src/components/Search/SearchBar/searchInput/valueContainerWithIcon.tsx b/web/apps/photos/src/components/Search/SearchBar/searchInput/valueContainerWithIcon.tsx index de9558777b..75529a925f 100644 --- a/web/apps/photos/src/components/Search/SearchBar/searchInput/valueContainerWithIcon.tsx +++ b/web/apps/photos/src/components/Search/SearchBar/searchInput/valueContainerWithIcon.tsx @@ -1,3 +1,7 @@ +import { + SearchOption, + SuggestionType, +} from "@/new/photos/services/search/types"; import { FlexWrapper } from "@ente/shared/components/Container"; import CalendarIcon from "@mui/icons-material/CalendarMonth"; import FolderIcon from "@mui/icons-material/Folder"; @@ -7,7 +11,6 @@ import SearchIcon from "@mui/icons-material/SearchOutlined"; import { Box } from "@mui/material"; import { components } from "react-select"; import { SelectComponents } from "react-select/src/components"; -import { SearchOption, SuggestionType } from "types/search"; const { ValueContainer } = components; diff --git a/web/apps/photos/src/components/Search/SearchResultInfo.tsx b/web/apps/photos/src/components/Search/SearchResultInfo.tsx index 7d99697bf1..96d610f6b1 100644 --- a/web/apps/photos/src/components/Search/SearchResultInfo.tsx +++ b/web/apps/photos/src/components/Search/SearchResultInfo.tsx @@ -1,8 +1,8 @@ +import { SearchResultSummary } from "@/new/photos/services/search/types"; import { Typography } from "@mui/material"; import { CollectionInfo } from "components/Collections/CollectionInfo"; import { CollectionInfoBarWrapper } from "components/Collections/styledComponents"; import { t } from "i18next"; -import { SearchResultSummary } from "types/search"; interface Iprops { searchResultSummary: SearchResultSummary; diff --git a/web/apps/photos/src/components/pages/gallery/Navbar.tsx b/web/apps/photos/src/components/pages/gallery/Navbar.tsx index a2cf98b4e3..dfb8a6339c 100644 --- a/web/apps/photos/src/components/pages/gallery/Navbar.tsx +++ b/web/apps/photos/src/components/pages/gallery/Navbar.tsx @@ -1,4 +1,5 @@ import { NavbarBase } from "@/base/components/Navbar"; +import { UpdateSearch } from "@/new/photos/services/search/types"; import { EnteFile } from "@/new/photos/types/file"; import { FlexWrapper, HorizontalFlex } from "@ente/shared/components/Container"; import ArrowBack from "@mui/icons-material/ArrowBack"; @@ -8,7 +9,6 @@ import SearchBar from "components/Search/SearchBar"; import UploadButton from "components/Upload/UploadButton"; import { t } from "i18next"; import { Collection } from "types/collection"; -import { UpdateSearch } from "types/search"; interface Iprops { openSidebar: () => void; diff --git a/web/apps/photos/src/pages/_app.tsx b/web/apps/photos/src/pages/_app.tsx index 53acd2e386..3cee4ec6ec 100644 --- a/web/apps/photos/src/pages/_app.tsx +++ b/web/apps/photos/src/pages/_app.tsx @@ -4,13 +4,13 @@ import { CustomHead } from "@/base/components/Head"; import { AppNavbar } from "@/base/components/Navbar"; import { setupI18n } from "@/base/i18n"; import log from "@/base/log"; -import { runMigrations } from "@/new/photos/services/migrations"; import { logStartupBanner, logUnhandledErrorsAndRejections, } from "@/base/log-web"; import { AppUpdate } from "@/base/types/ipc"; import DownloadManager from "@/new/photos/services/download"; +import { runMigrations } from "@/new/photos/services/migrations"; import { initML, isMLSupported } from "@/new/photos/services/ml"; import { ensure } from "@/utils/ensure"; import { Overlay } from "@ente/shared/components/Container"; diff --git a/web/apps/photos/src/pages/gallery/index.tsx b/web/apps/photos/src/pages/gallery/index.tsx index 74273eb014..cd524d3f1e 100644 --- a/web/apps/photos/src/pages/gallery/index.tsx +++ b/web/apps/photos/src/pages/gallery/index.tsx @@ -7,6 +7,12 @@ import { getLocalTrashedFiles, } from "@/new/photos/services/files"; import { wipHasSwitchedOnceCmpAndSet } from "@/new/photos/services/ml"; +import { search, setSearchableFiles } from "@/new/photos/services/search"; +import { + SearchQuery, + SearchResultSummary, + UpdateSearch, +} from "@/new/photos/services/search/types"; import { EnteFile } from "@/new/photos/types/file"; import { mergeMetadata } from "@/new/photos/utils/file"; import { CenteredFlex } from "@ente/shared/components/Container"; @@ -31,7 +37,6 @@ import { getKey, } from "@ente/shared/storage/sessionStorage"; import type { User } from "@ente/shared/user/types"; -import { isPromise } from "@ente/shared/utils"; import { Typography, styled } from "@mui/material"; import AuthenticateUserModal from "components/AuthenticateUserModal"; import Collections from "components/Collections"; @@ -106,7 +111,6 @@ import { SetFilesDownloadProgressAttributes, SetFilesDownloadProgressAttributesCreator, } from "types/gallery"; -import { Search, SearchResultSummary, UpdateSearch } from "types/search"; import { FamilyData } from "types/user"; import { checkSubscriptionPurchase } from "utils/billing"; import { @@ -119,7 +123,6 @@ import { hasNonSystemCollections, splitNormalAndHiddenCollections, } from "utils/collection"; -import ComlinkSearchWorker from "utils/comlink/ComlinkSearchWorker"; import { FILE_OPS_TYPE, constructFileToCollectionMap, @@ -193,7 +196,7 @@ export default function Gallery() { const [collectionNamerAttributes, setCollectionNamerAttributes] = useState(null); const [collectionNamerView, setCollectionNamerView] = useState(false); - const [search, setSearch] = useState(null); + const [searchQuery, setSearchQuery] = useState(null); const [shouldDisableDropzone, setShouldDisableDropzone] = useState(false); const [isPhotoSwipeOpen, setIsPhotoSwipeOpen] = useState(false); // TODO(MR): This is never true currently, this is the WIP ability to show @@ -400,13 +403,7 @@ export default function Gallery() { }; }, []); - useEffectSingleThreaded( - async ([files]: [files: EnteFile[]]) => { - const searchWorker = await ComlinkSearchWorker.getInstance(); - await searchWorker.setFiles(files); - }, - [files], - ); + useEffect(() => setSearchableFiles(files), [files]); useEffect(() => { if (!user || !files || !collections || !hiddenFiles || !trashedFiles) { @@ -523,11 +520,9 @@ export default function Gallery() { ]); } - const searchWorker = await ComlinkSearchWorker.getInstance(); - let filteredFiles: EnteFile[] = []; if (isInSearchMode) { - filteredFiles = getUniqueFiles(await searchWorker.search(search)); + filteredFiles = getUniqueFiles(await search(searchQuery)); } else { filteredFiles = getUniqueFiles( (isInHiddenSection ? hiddenFiles : files).filter((item) => { @@ -587,9 +582,9 @@ export default function Gallery() { }), ); } - if (search?.clip) { + if (searchQuery?.clip) { return filteredFiles.sort((a, b) => { - return search.clip.get(b.id) - search.clip.get(a.id); + return searchQuery.clip.get(b.id) - searchQuery.clip.get(a.id); }); } const sortAsc = activeCollection?.pubMagicMetadata?.data?.asc ?? false; @@ -605,7 +600,7 @@ export default function Gallery() { tempDeletedFileIds, tempHiddenFileIds, hiddenFileIds, - search, + searchQuery, activeCollectionID, archivedCollections, ]); @@ -975,7 +970,7 @@ export default function Gallery() { if (newSearch?.collection) { setActiveCollectionID(newSearch?.collection); } else { - setSearch(newSearch); + setSearchQuery(newSearch); } setIsClipSearchResult(!!newSearch?.clip); if (!newSearch?.collection) { @@ -1243,37 +1238,6 @@ export default function Gallery() { ); } -// useEffectSingleThreaded is a useEffect that will only run one at a time, and will -// caches the latest deps of requests that come in while it is running, and will -// run that after the current run is complete. -function useEffectSingleThreaded( - fn: (deps) => void | Promise, - deps: any[], -): void { - const updateInProgress = useRef(false); - const nextRequestDepsRef = useRef(null); - useEffect(() => { - const main = async (deps) => { - if (updateInProgress.current) { - nextRequestDepsRef.current = deps; - return; - } - updateInProgress.current = true; - const result = fn(deps); - if (isPromise(result)) { - await result; - } - updateInProgress.current = false; - if (nextRequestDepsRef.current) { - const deps = nextRequestDepsRef.current; - nextRequestDepsRef.current = null; - setTimeout(() => main(deps), 0); - } - }; - main(deps); - }, deps); -} - /** * Preload all three variants of a responsive image. */ diff --git a/web/apps/photos/src/services/heic-convert.ts b/web/apps/photos/src/services/heic-convert.ts deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/web/apps/photos/src/services/locationSearchService.ts b/web/apps/photos/src/services/locationSearchService.ts index 30a36e5e04..9e3ff032d3 100644 --- a/web/apps/photos/src/services/locationSearchService.ts +++ b/web/apps/photos/src/services/locationSearchService.ts @@ -1,15 +1,5 @@ import log from "@/base/log"; -import type { Location, LocationTagData } from "types/entity"; - -export interface City { - city: string; - country: string; - lat: number; - lng: number; -} - -const DEFAULT_CITY_RADIUS = 10; -const KMS_PER_DEGREE = 111.16; +import type { City } from "@/new/photos/services/search/types"; class LocationSearchService { private cities: Array = []; @@ -53,45 +43,3 @@ class LocationSearchService { } export default new LocationSearchService(); - -export function isInsideLocationTag( - location: Location, - locationTag: LocationTagData, -) { - return isLocationCloseToPoint( - location, - locationTag.centerPoint, - locationTag.radius, - ); -} - -export function isInsideCity(location: Location, city: City) { - return isLocationCloseToPoint( - { latitude: city.lat, longitude: city.lng }, - location, - DEFAULT_CITY_RADIUS, - ); -} - -function isLocationCloseToPoint( - centerPoint: Location, - location: Location, - radius: number, -) { - const a = (radius * _scaleFactor(centerPoint.latitude)) / KMS_PER_DEGREE; - const b = radius / KMS_PER_DEGREE; - const x = centerPoint.latitude - location.latitude; - const y = centerPoint.longitude - location.longitude; - if ((x * x) / (a * a) + (y * y) / (b * b) <= 1) { - return true; - } - return false; -} - -///The area bounded by the location tag becomes more elliptical with increase -///in the magnitude of the latitude on the caritesian plane. When latitude is -///0 degrees, the ellipse is a circle with a = b = r. When latitude incrases, -///the major axis (a) has to be scaled by the secant of the latitude. -function _scaleFactor(lat: number) { - return 1 / Math.cos(lat * (Math.PI / 180)); -} diff --git a/web/apps/photos/src/services/searchService.ts b/web/apps/photos/src/services/searchService.ts index 76b1d5cb2b..cf50458926 100644 --- a/web/apps/photos/src/services/searchService.ts +++ b/web/apps/photos/src/services/searchService.ts @@ -8,26 +8,27 @@ import { mlStatusSnapshot, wipSearchPersons, } from "@/new/photos/services/ml"; -import { parseDateComponents } from "@/new/photos/services/search"; +import { parseDateComponents, search } from "@/new/photos/services/search"; import type { SearchDateComponents, SearchPerson, } from "@/new/photos/services/search/types"; +import { + City, + ClipSearchScores, + LocationTagData, + SearchOption, + SearchQuery, + Suggestion, + SuggestionType, +} from "@/new/photos/services/search/types"; import { EnteFile } from "@/new/photos/types/file"; import { t } from "i18next"; import { Collection } from "types/collection"; -import { EntityType, LocationTag, LocationTagData } from "types/entity"; -import { - ClipSearchScores, - Search, - SearchOption, - Suggestion, - SuggestionType, -} from "types/search"; -import ComlinkSearchWorker from "utils/comlink/ComlinkSearchWorker"; +import { EntityType, LocationTag } from "types/entity"; import { getUniqueFiles } from "utils/file"; import { getLatestEntities } from "./entityService"; -import locationSearchService, { City } from "./locationSearchService"; +import locationSearchService from "./locationSearchService"; export const getDefaultOptions = async () => { return [ @@ -64,13 +65,10 @@ export const getAutoCompleteSuggestions = async function convertSuggestionsToOptions( suggestions: Suggestion[], ): Promise { - const searchWorker = await ComlinkSearchWorker.getInstance(); const previewImageAppendedOptions: SearchOption[] = []; for (const suggestion of suggestions) { const searchQuery = convertSuggestionToSearchQuery(suggestion); - const resultFiles = getUniqueFiles( - await searchWorker.search(searchQuery), - ); + const resultFiles = getUniqueFiles(await search(searchQuery)); if (searchQuery?.clip) { resultFiles.sort((a, b) => { const aScore = searchQuery.clip.get(a.id); @@ -304,7 +302,7 @@ const searchClip = async ( return matches; }; -function convertSuggestionToSearchQuery(option: Suggestion): Search { +function convertSuggestionToSearchQuery(option: Suggestion): SearchQuery { switch (option.type) { case SuggestionType.DATE: return { diff --git a/web/apps/photos/src/types/entity.ts b/web/apps/photos/src/types/entity.ts index 4371562cc8..f22a7bac8a 100644 --- a/web/apps/photos/src/types/entity.ts +++ b/web/apps/photos/src/types/entity.ts @@ -1,3 +1,5 @@ +import type { LocationTagData } from "@/new/photos/services/search/types"; + export enum EntityType { LOCATION_TAG = "location", } @@ -25,19 +27,6 @@ export interface EncryptedEntity { userID: number; } -export interface Location { - latitude: number | null; - longitude: number | null; -} - -export interface LocationTagData { - name: string; - radius: number; - aSquare: number; - bSquare: number; - centerPoint: Location; -} - export type LocationTag = Entity; export interface Entity diff --git a/web/apps/photos/src/types/search/index.ts b/web/apps/photos/src/types/search/index.ts index 4c0b901655..e69de29bb2 100644 --- a/web/apps/photos/src/types/search/index.ts +++ b/web/apps/photos/src/types/search/index.ts @@ -1,65 +0,0 @@ -import { FileType } from "@/media/file-type"; -import type { MLStatus } from "@/new/photos/services/ml"; -import type { - SearchDateComponents, - SearchPerson, -} from "@/new/photos/services/search/types"; -import { EnteFile } from "@/new/photos/types/file"; -import { City } from "services/locationSearchService"; -import { LocationTagData } from "types/entity"; - -export enum SuggestionType { - DATE = "DATE", - LOCATION = "LOCATION", - COLLECTION = "COLLECTION", - FILE_NAME = "FILE_NAME", - PERSON = "PERSON", - INDEX_STATUS = "INDEX_STATUS", - FILE_CAPTION = "FILE_CAPTION", - FILE_TYPE = "FILE_TYPE", - CLIP = "CLIP", - CITY = "CITY", -} - -export interface Suggestion { - type: SuggestionType; - label: string; - value: - | SearchDateComponents - | number[] - | SearchPerson - | MLStatus - | LocationTagData - | City - | FileType - | ClipSearchScores; - hide?: boolean; -} - -export type Search = { - date?: SearchDateComponents; - location?: LocationTagData; - city?: City; - collection?: number; - files?: number[]; - person?: SearchPerson; - fileType?: FileType; - clip?: ClipSearchScores; -}; - -export type SearchResultSummary = { - optionName: string; - fileCount: number; -}; - -export interface SearchOption extends Suggestion { - fileCount: number; - previewFiles: EnteFile[]; -} - -export type UpdateSearch = ( - search: Search, - summary: SearchResultSummary, -) => void; - -export type ClipSearchScores = Map; diff --git a/web/apps/photos/src/utils/comlink/ComlinkSearchWorker.ts b/web/apps/photos/src/utils/comlink/ComlinkSearchWorker.ts deleted file mode 100644 index 7ba6a67bf3..0000000000 --- a/web/apps/photos/src/utils/comlink/ComlinkSearchWorker.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { haveWindow } from "@/base/env"; -import { ComlinkWorker } from "@/base/worker/comlink-worker"; -import type { Remote } from "comlink"; -import { type DedicatedSearchWorker } from "worker/search.worker"; - -class ComlinkSearchWorker { - private comlinkWorkerInstance: Remote; - private comlinkWorker: ComlinkWorker; - - async getInstance() { - if (!this.comlinkWorkerInstance) { - if (!this.comlinkWorker) - this.comlinkWorker = getDedicatedSearchWorker(); - this.comlinkWorkerInstance = await this.comlinkWorker.remote; - } - return this.comlinkWorkerInstance; - } -} - -export const getDedicatedSearchWorker = () => { - if (haveWindow()) { - const cryptoComlinkWorker = new ComlinkWorker< - typeof DedicatedSearchWorker - >( - "ente-search-worker", - new Worker(new URL("worker/search.worker.ts", import.meta.url)), - ); - return cryptoComlinkWorker; - } -}; - -export default new ComlinkSearchWorker(); diff --git a/web/apps/photos/src/worker/search.worker.ts b/web/apps/photos/src/worker/search.worker.ts deleted file mode 100644 index 1100e3c223..0000000000 --- a/web/apps/photos/src/worker/search.worker.ts +++ /dev/null @@ -1,88 +0,0 @@ -import { getUICreationDate } from "@/media/file-metadata"; -import type { SearchDateComponents } from "@/new/photos/services/search/types"; -import { EnteFile } from "@/new/photos/types/file"; -import { getPublicMagicMetadataSync } from "@ente/shared/file-metadata"; -import * as Comlink from "comlink"; -import { - isInsideCity, - isInsideLocationTag, -} from "services/locationSearchService"; -import { Search } from "types/search"; - -export class DedicatedSearchWorker { - private files: EnteFile[] = []; - - setFiles(files: EnteFile[]) { - this.files = files; - } - - search(search: Search) { - return this.files.filter((file) => { - return isSearchedFile(file, search); - }); - } -} - -Comlink.expose(DedicatedSearchWorker, self); - -function isSearchedFile(file: EnteFile, search: Search) { - if (search?.collection) { - return search.collection === file.collectionID; - } - - if (search?.date) { - return isDateComponentsMatch( - search.date, - getUICreationDate(file, getPublicMagicMetadataSync(file)), - ); - } - if (search?.location) { - return isInsideLocationTag( - { - latitude: file.metadata.latitude, - longitude: file.metadata.longitude, - }, - search.location, - ); - } - if (search?.city) { - return isInsideCity( - { - latitude: file.metadata.latitude, - longitude: file.metadata.longitude, - }, - search.city, - ); - } - if (search?.files) { - return search.files.indexOf(file.id) !== -1; - } - if (search?.person) { - return search.person.files.indexOf(file.id) !== -1; - } - if (typeof search?.fileType !== "undefined") { - return search.fileType === file.metadata.fileType; - } - if (typeof search?.clip !== "undefined") { - return search.clip.has(file.id); - } - return false; -} - -const isDateComponentsMatch = ( - { year, month, day, weekday, hour }: SearchDateComponents, - date: Date, -) => { - // Components are guaranteed to have at least one attribute present, so - // start by assuming true. - let match = true; - - if (year) match = date.getFullYear() == year; - // JS getMonth is 0-indexed. - if (match && month) match = date.getMonth() + 1 == month; - if (match && day) match = date.getDate() == day; - if (match && weekday) match = date.getDay() == weekday; - if (match && hour) match = date.getHours() == hour; - - return match; -}; diff --git a/web/packages/new/photos/services/search/index.ts b/web/packages/new/photos/services/search/index.ts index d5d3154f5e..4c9cb9550d 100644 --- a/web/packages/new/photos/services/search/index.ts +++ b/web/packages/new/photos/services/search/index.ts @@ -2,7 +2,44 @@ import { nullToUndefined } from "@/utils/transform"; import type { Component } from "chrono-node"; import * as chrono from "chrono-node"; import i18n, { t } from "i18next"; -import type { SearchDateComponents } from "./types"; +import type { SearchDateComponents, SearchQuery } from "./types"; + +import { ComlinkWorker } from "@/base/worker/comlink-worker"; +import type { EnteFile } from "../../types/file"; +import type { SearchWorker } from "./worker"; + +/** + * Cached instance of the {@link ComlinkWorker} that wraps our web worker. + */ +let _comlinkWorker: ComlinkWorker | undefined; + +/** + * Lazily created, cached, instance of {@link SearchWorker}. + */ +const worker = () => (_comlinkWorker ??= createComlinkWorker()).remote; + +/** + * Create a new instance of a comlink worker that wraps a {@link SearchWorker} + * web worker. + */ +const createComlinkWorker = () => + new ComlinkWorker( + "search", + new Worker(new URL("worker.ts", import.meta.url)), + ); + +/** + * Set the files over which we will search. + */ +export const setSearchableFiles = (enteFiles: EnteFile[]) => + void worker().then((w) => w.setEnteFiles(enteFiles)); + +/** + * Search for and return the list of {@link EnteFile}s that match the given + * {@link search} query. + */ +export const search = async (search: SearchQuery) => + worker().then((w) => w.search(search)); interface DateSearchResult { components: SearchDateComponents; diff --git a/web/packages/new/photos/services/search/types.ts b/web/packages/new/photos/services/search/types.ts index f80a4b4565..03cd7a504b 100644 --- a/web/packages/new/photos/services/search/types.ts +++ b/web/packages/new/photos/services/search/types.ts @@ -1,9 +1,11 @@ /** * @file types shared between the main thread interface to search (`index.ts`) - * and the search worker (`worker.ts`) + * and the search worker that does the actual searching (`worker.ts`). */ -import type { EnteFile } from "../../types/file"; +import { FileType } from "@/media/file-type"; +import type { MLStatus } from "@/new/photos/services/ml"; +import type { EnteFile } from "@/new/photos/types/file"; /** * A parsed version of a potential natural language date time string. @@ -48,3 +50,81 @@ export interface SearchPerson { displayFaceID: string; displayFaceFile: EnteFile; } + +// TODO-cgroup: Audit below + +export interface Location { + latitude: number | null; + longitude: number | null; +} + +export interface LocationTagData { + name: string; + radius: number; + aSquare: number; + bSquare: number; + centerPoint: Location; +} + +export interface City { + city: string; + country: string; + lat: number; + lng: number; +} + +export enum SuggestionType { + DATE = "DATE", + LOCATION = "LOCATION", + COLLECTION = "COLLECTION", + FILE_NAME = "FILE_NAME", + PERSON = "PERSON", + INDEX_STATUS = "INDEX_STATUS", + FILE_CAPTION = "FILE_CAPTION", + FILE_TYPE = "FILE_TYPE", + CLIP = "CLIP", + CITY = "CITY", +} + +export interface Suggestion { + type: SuggestionType; + label: string; + value: + | SearchDateComponents + | number[] + | SearchPerson + | MLStatus + | LocationTagData + | City + | FileType + | ClipSearchScores; + hide?: boolean; +} + +export interface SearchQuery { + date?: SearchDateComponents; + location?: LocationTagData; + city?: City; + collection?: number; + files?: number[]; + person?: SearchPerson; + fileType?: FileType; + clip?: ClipSearchScores; +} + +export interface SearchResultSummary { + optionName: string; + fileCount: number; +} + +export interface SearchOption extends Suggestion { + fileCount: number; + previewFiles: EnteFile[]; +} + +export type UpdateSearch = ( + search: SearchQuery, + summary: SearchResultSummary, +) => void; + +export type ClipSearchScores = Map; diff --git a/web/packages/new/photos/services/search/worker.ts b/web/packages/new/photos/services/search/worker.ts new file mode 100644 index 0000000000..549477d8fb --- /dev/null +++ b/web/packages/new/photos/services/search/worker.ts @@ -0,0 +1,146 @@ +// TODO-cgroups +/* eslint-disable @typescript-eslint/no-non-null-assertion */ +/* eslint-disable @typescript-eslint/prefer-includes */ +/* eslint-disable @typescript-eslint/no-unnecessary-condition */ +import { getUICreationDate } from "@/media/file-metadata"; +import type { + City, + Location, + LocationTagData, +} from "@/new/photos/services/search/types"; +import type { EnteFile } from "@/new/photos/types/file"; +import { getPublicMagicMetadataSync } from "@ente/shared/file-metadata"; +import { expose } from "comlink"; +import type { SearchDateComponents, SearchQuery } from "./types"; + +/** + * A web worker that runs the search asynchronously so that the main thread + * remains responsive. + */ +export class SearchWorker { + private enteFiles: EnteFile[] = []; + + /** + * Set the files that we should search across. + */ + setEnteFiles(enteFiles: EnteFile[]) { + this.enteFiles = enteFiles; + } + + /** + * Return {@link EnteFile}s that satisfy the given {@link searchQuery} + * query. + */ + search(searchQuery: SearchQuery) { + return this.enteFiles.filter((f) => isMatch(f, searchQuery)); + } +} + +expose(SearchWorker); + +function isMatch(file: EnteFile, query: SearchQuery) { + if (query?.collection) { + return query.collection === file.collectionID; + } + + if (query?.date) { + return isDateComponentsMatch( + query.date, + getUICreationDate(file, getPublicMagicMetadataSync(file)), + ); + } + if (query?.location) { + return isInsideLocationTag( + { + latitude: file.metadata.latitude ?? null, + longitude: file.metadata.longitude ?? null, + }, + query.location, + ); + } + if (query?.city) { + return isInsideCity( + { + latitude: file.metadata.latitude ?? null, + longitude: file.metadata.longitude ?? null, + }, + query.city, + ); + } + if (query?.files) { + return query.files.indexOf(file.id) !== -1; + } + if (query?.person) { + return query.person.files.indexOf(file.id) !== -1; + } + if (typeof query?.fileType !== "undefined") { + return query.fileType === file.metadata.fileType; + } + if (typeof query?.clip !== "undefined") { + return query.clip.has(file.id); + } + return false; +} + +const isDateComponentsMatch = ( + { year, month, day, weekday, hour }: SearchDateComponents, + date: Date, +) => { + // Components are guaranteed to have at least one attribute present, so + // start by assuming true. + let match = true; + + if (year) match = date.getFullYear() == year; + // JS getMonth is 0-indexed. + if (match && month) match = date.getMonth() + 1 == month; + if (match && day) match = date.getDate() == day; + if (match && weekday) match = date.getDay() == weekday; + if (match && hour) match = date.getHours() == hour; + + return match; +}; + +export function isInsideLocationTag( + location: Location, + locationTag: LocationTagData, +) { + return isLocationCloseToPoint( + location, + locationTag.centerPoint, + locationTag.radius, + ); +} + +const DEFAULT_CITY_RADIUS = 10; +const KMS_PER_DEGREE = 111.16; + +export function isInsideCity(location: Location, city: City) { + return isLocationCloseToPoint( + { latitude: city.lat, longitude: city.lng }, + location, + DEFAULT_CITY_RADIUS, + ); +} + +function isLocationCloseToPoint( + centerPoint: Location, + location: Location, + radius: number, +) { + const a = (radius * _scaleFactor(centerPoint.latitude!)) / KMS_PER_DEGREE; + const b = radius / KMS_PER_DEGREE; + const x = centerPoint.latitude! - location.latitude!; + const y = centerPoint.longitude! - location.longitude!; + if ((x * x) / (a * a) + (y * y) / (b * b) <= 1) { + return true; + } + return false; +} + +///The area bounded by the location tag becomes more elliptical with increase +///in the magnitude of the latitude on the caritesian plane. When latitude is +///0 degrees, the ellipse is a circle with a = b = r. When latitude incrases, +///the major axis (a) has to be scaled by the secant of the latitude. +function _scaleFactor(lat: number) { + return 1 / Math.cos(lat * (Math.PI / 180)); +} diff --git a/web/packages/shared/file-metadata.ts b/web/packages/shared/file-metadata.ts index 0723a975ef..724fc3740b 100644 --- a/web/packages/shared/file-metadata.ts +++ b/web/packages/shared/file-metadata.ts @@ -3,7 +3,7 @@ import { decryptPublicMagicMetadata, type PublicMagicMetadata, } from "@/media/file-metadata"; -import { EnteFile } from "@/new/photos/types/file"; +import type { EnteFile } from "@/new/photos/types/file"; import { fileLogID } from "@/new/photos/utils/file"; /** From f958b16343d4423c1e8a6b996819a17e30b4206a Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 6 Sep 2024 10:36:44 +0530 Subject: [PATCH 16/42] Rearrange --- web/apps/photos/src/services/searchService.ts | 10 +- .../new/photos/services/search/index.ts | 95 +++--------- .../new/photos/services/search/types.ts | 5 + .../new/photos/services/search/worker.ts | 138 +++++++++++++++++- 4 files changed, 159 insertions(+), 89 deletions(-) diff --git a/web/apps/photos/src/services/searchService.ts b/web/apps/photos/src/services/searchService.ts index cf50458926..f869b1e931 100644 --- a/web/apps/photos/src/services/searchService.ts +++ b/web/apps/photos/src/services/searchService.ts @@ -30,6 +30,8 @@ import { getUniqueFiles } from "utils/file"; import { getLatestEntities } from "./entityService"; import locationSearchService from "./locationSearchService"; +// Suggestions shown in the search dropdown's empty state, i.e. when the user +// selects the search bar but does not provide any input. export const getDefaultOptions = async () => { return [ await getMLStatusSuggestion(), @@ -37,6 +39,7 @@ export const getDefaultOptions = async () => { ].filter((t) => !!t); }; +// Suggestions shown in the search dropdown when the user has typed something. export const getAutoCompleteSuggestions = (files: EnteFile[], collections: Collection[]) => async (searchPhrase: string): Promise => { @@ -157,13 +160,6 @@ export async function getMLStatusSuggestion(): Promise { }; } -const getDateSuggestion = (searchPhrase: string): Suggestion[] => - parseDateComponents(searchPhrase).map(({ components, label }) => ({ - type: SuggestionType.DATE, - value: components, - label, - })); - function getCollectionSuggestion( searchPhrase: string, collections: Collection[], diff --git a/web/packages/new/photos/services/search/index.ts b/web/packages/new/photos/services/search/index.ts index 4c9cb9550d..8f1919974e 100644 --- a/web/packages/new/photos/services/search/index.ts +++ b/web/packages/new/photos/services/search/index.ts @@ -1,11 +1,7 @@ -import { nullToUndefined } from "@/utils/transform"; -import type { Component } from "chrono-node"; -import * as chrono from "chrono-node"; -import i18n, { t } from "i18next"; -import type { SearchDateComponents, SearchQuery } from "./types"; - import { ComlinkWorker } from "@/base/worker/comlink-worker"; +import i18n, { t } from "i18next"; import type { EnteFile } from "../../types/file"; +import type { DateSearchResult, SearchQuery } from "./types"; import type { SearchWorker } from "./worker"; /** @@ -34,6 +30,17 @@ const createComlinkWorker = () => export const setSearchableFiles = (enteFiles: EnteFile[]) => void worker().then((w) => w.setEnteFiles(enteFiles)); +/** + * Convert a search string into a reusable "search query" that can be passed on + * to the {@link search} function. + * + * @param searchString The string we want to search for. + */ +export const createSearchQuery = (searchString: string) => + worker().then((w) => + w.createSearchQuery(searchString, i18n.language, holidays()), + ); + /** * Search for and return the list of {@link EnteFile}s that match the given * {@link search} query. @@ -41,83 +48,21 @@ export const setSearchableFiles = (enteFiles: EnteFile[]) => export const search = async (search: SearchQuery) => worker().then((w) => w.search(search)); -interface DateSearchResult { - components: SearchDateComponents; - label: string; -} - /** - * Try to parse an arbitrary search string into sets of date components. + * A list of holidays - their yearly dates and localized names. * - * e.g. "December 2022" will be parsed into a + * --- * - * [(year 2022, month 12, day undefined)] + * We keep this on the main thread since it uses the t() function for + * localization (although I haven't tried that in a web worker, it might work + * there too). * - * while "22 December 2022" will be parsed into - * - * [(year 2022, month 12, day 22)] - * - * In addition, also return a formatted representation of the "best" guess at - * the date that was intended by the search string. + * Also, this cannot be a const since it needs to be evaluated lazily for the + * t() to work. */ -export const parseDateComponents = (s: string): DateSearchResult[] => - parseChrono(s) - .concat(parseYearComponents(s)) - .concat(parseHolidayComponents(s)); - -export const parseChrono = (s: string): DateSearchResult[] => - chrono - .parse(s) - .map((result) => { - const p = result.start; - const component = (s: Component) => - p.isCertain(s) ? nullToUndefined(p.get(s)) : undefined; - - const year = component("year"); - const month = component("month"); - const day = component("day"); - const weekday = component("weekday"); - const hour = component("hour"); - - if (!year && !month && !day && !weekday && !hour) return undefined; - const components = { year, month, day, weekday, hour }; - - const format: Intl.DateTimeFormatOptions = {}; - if (year) format.year = "numeric"; - if (month) format.month = "long"; - if (day) format.day = "numeric"; - if (weekday) format.weekday = "long"; - if (hour) { - format.hour = "numeric"; - format.dayPeriod = "short"; - } - - const formatter = new Intl.DateTimeFormat(i18n.language, format); - const label = formatter.format(p.date()); - return { components, label }; - }) - .filter((x) => x !== undefined); - -/** chrono does not parse years like "2024", so do it manually. */ -const parseYearComponents = (s: string): DateSearchResult[] => { - // s is already trimmed. - if (s.length == 4) { - const year = parseInt(s); - if (year && year <= 9999) { - const components = { year }; - return [{ components, label: s }]; - } - } - return []; -}; - -// This cannot be a const, it needs to be evaluated lazily for the t() to work. const holidays = (): DateSearchResult[] => [ { components: { month: 12, day: 25 }, label: t("CHRISTMAS") }, { components: { month: 12, day: 24 }, label: t("CHRISTMAS_EVE") }, { components: { month: 1, day: 1 }, label: t("NEW_YEAR") }, { components: { month: 12, day: 31 }, label: t("NEW_YEAR_EVE") }, ]; - -const parseHolidayComponents = (s: string) => - holidays().filter(({ label }) => label.toLowerCase().includes(s)); diff --git a/web/packages/new/photos/services/search/types.ts b/web/packages/new/photos/services/search/types.ts index 03cd7a504b..a80799ab0e 100644 --- a/web/packages/new/photos/services/search/types.ts +++ b/web/packages/new/photos/services/search/types.ts @@ -7,6 +7,11 @@ import { FileType } from "@/media/file-type"; import type { MLStatus } from "@/new/photos/services/ml"; import type { EnteFile } from "@/new/photos/types/file"; +export interface DateSearchResult { + components: SearchDateComponents; + label: string; +} + /** * A parsed version of a potential natural language date time string. * diff --git a/web/packages/new/photos/services/search/worker.ts b/web/packages/new/photos/services/search/worker.ts index 549477d8fb..5f19f91dba 100644 --- a/web/packages/new/photos/services/search/worker.ts +++ b/web/packages/new/photos/services/search/worker.ts @@ -3,15 +3,22 @@ /* eslint-disable @typescript-eslint/prefer-includes */ /* eslint-disable @typescript-eslint/no-unnecessary-condition */ import { getUICreationDate } from "@/media/file-metadata"; +import type { EnteFile } from "@/new/photos/types/file"; +import { wait } from "@/utils/promise"; +import { nullToUndefined } from "@/utils/transform"; +import { getPublicMagicMetadataSync } from "@ente/shared/file-metadata"; +import type { Component } from "chrono-node"; +import * as chrono from "chrono-node"; +import { expose } from "comlink"; import type { City, Location, LocationTagData, -} from "@/new/photos/services/search/types"; -import type { EnteFile } from "@/new/photos/types/file"; -import { getPublicMagicMetadataSync } from "@ente/shared/file-metadata"; -import { expose } from "comlink"; -import type { SearchDateComponents, SearchQuery } from "./types"; + SearchDateComponents, + SearchQuery, + Suggestion, +} from "./types"; +import { SuggestionType } from "./types"; /** * A web worker that runs the search asynchronously so that the main thread @@ -28,8 +35,18 @@ export class SearchWorker { } /** - * Return {@link EnteFile}s that satisfy the given {@link searchQuery} - * query. + * Convert a search string into a reusable query. + */ + async createSearchQuery( + searchString: string, + locale: string, + holidays: DateSearchResult[], + ) { + return createSearchQuery(searchString, locale, holidays); + } + + /** + * Return {@link EnteFile}s that satisfy the given {@link searchQuery}. */ search(searchQuery: SearchQuery) { return this.enteFiles.filter((f) => isMatch(f, searchQuery)); @@ -38,6 +55,113 @@ export class SearchWorker { expose(SearchWorker); +const createSearchQuery = async ( + searchString: string, + locale: string, + holidays: DateSearchResult[], +): Promise => { + // Normalize it by trimming whitespace and converting to lowercase. + const s = searchString.trim().toLowerCase(); + if (s.length == 0) return []; + + // TODO Temp + await wait(0); + return [dateSuggestion(s, locale, holidays)].flat(); +}; + +const dateSuggestion = ( + s: string, + locale: string, + holidays: DateSearchResult[], +) => + parseDateComponents(s, locale, holidays).map(({ components, label }) => ({ + type: SuggestionType.DATE, + value: components, + label, + })); + +interface DateSearchResult { + components: SearchDateComponents; + label: string; +} + +/** + * Try to parse an arbitrary search string into sets of date components. + * + * e.g. "December 2022" will be parsed into a + * + * [(year 2022, month 12, day undefined)] + * + * while "22 December 2022" will be parsed into + * + * [(year 2022, month 12, day 22)] + * + * In addition, also return a formatted representation of the "best" guess at + * the date that was intended by the search string. + */ +export const parseDateComponents = ( + s: string, + locale: string, + holidays: DateSearchResult[], +): DateSearchResult[] => + [ + parseChrono(s, locale), + parseYearComponents(s), + parseHolidayComponents(s, holidays), + ].flat(); + +const parseChrono = (s: string, locale: string): DateSearchResult[] => + chrono + .parse(s) + .map((result) => { + const p = result.start; + const component = (s: Component) => + p.isCertain(s) ? nullToUndefined(p.get(s)) : undefined; + + const year = component("year"); + const month = component("month"); + const day = component("day"); + const weekday = component("weekday"); + const hour = component("hour"); + + if (!year && !month && !day && !weekday && !hour) return undefined; + const components = { year, month, day, weekday, hour }; + + const format: Intl.DateTimeFormatOptions = {}; + if (year) format.year = "numeric"; + if (month) format.month = "long"; + if (day) format.day = "numeric"; + if (weekday) format.weekday = "long"; + if (hour) { + format.hour = "numeric"; + format.dayPeriod = "short"; + } + + // TODO Temp + console.log("locale", locale); + + const formatter = new Intl.DateTimeFormat(locale, format); + const label = formatter.format(p.date()); + return { components, label }; + }) + .filter((x) => x !== undefined); + +/** chrono does not parse years like "2024", so do it manually. */ +const parseYearComponents = (s: string): DateSearchResult[] => { + // s is already trimmed. + if (s.length == 4) { + const year = parseInt(s); + if (year && year <= 9999) { + const components = { year }; + return [{ components, label: s }]; + } + } + return []; +}; + +const parseHolidayComponents = (s: string, holidays: DateSearchResult[]) => + holidays.filter(({ label }) => label.toLowerCase().includes(s)); + function isMatch(file: EnteFile, query: SearchQuery) { if (query?.collection) { return query.collection === file.collectionID; From 5195e2ac74773c7aef950d5dd54f56b5029df025 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 6 Sep 2024 11:16:32 +0530 Subject: [PATCH 17/42] Move --- web/apps/photos/src/services/searchService.ts | 6 ++++-- web/packages/new/photos/services/search/index.ts | 10 +++------- web/packages/new/photos/services/search/worker.ts | 11 ++--------- 3 files changed, 9 insertions(+), 18 deletions(-) diff --git a/web/apps/photos/src/services/searchService.ts b/web/apps/photos/src/services/searchService.ts index f869b1e931..943edbbb47 100644 --- a/web/apps/photos/src/services/searchService.ts +++ b/web/apps/photos/src/services/searchService.ts @@ -8,7 +8,7 @@ import { mlStatusSnapshot, wipSearchPersons, } from "@/new/photos/services/ml"; -import { parseDateComponents, search } from "@/new/photos/services/search"; +import { createSearchQuery, search } from "@/new/photos/services/search"; import type { SearchDateComponents, SearchPerson, @@ -51,7 +51,9 @@ export const getAutoCompleteSuggestions = const suggestions: Suggestion[] = [ await getClipSuggestion(searchPhrase), ...getFileTypeSuggestion(searchPhrase), - ...getDateSuggestion(searchPhrase), + // The following functionality has moved to createSearchQuery + // - getDateSuggestion(searchPhrase), + ...(await createSearchQuery(searchPhrase)), ...getCollectionSuggestion(searchPhrase, collections), getFileNameSuggestion(searchPhrase, files), getFileCaptionSuggestion(searchPhrase, files), diff --git a/web/packages/new/photos/services/search/index.ts b/web/packages/new/photos/services/search/index.ts index 8f1919974e..6c6dcb5ee6 100644 --- a/web/packages/new/photos/services/search/index.ts +++ b/web/packages/new/photos/services/search/index.ts @@ -51,14 +51,10 @@ export const search = async (search: SearchQuery) => /** * A list of holidays - their yearly dates and localized names. * - * --- - * - * We keep this on the main thread since it uses the t() function for + * We need to keep this on the main thread since it uses the t() function for * localization (although I haven't tried that in a web worker, it might work - * there too). - * - * Also, this cannot be a const since it needs to be evaluated lazily for the - * t() to work. + * there too). Also, it cannot be a const since it needs to be evaluated lazily + * for the t() to work. */ const holidays = (): DateSearchResult[] => [ { components: { month: 12, day: 25 }, label: t("CHRISTMAS") }, diff --git a/web/packages/new/photos/services/search/worker.ts b/web/packages/new/photos/services/search/worker.ts index 5f19f91dba..032024c632 100644 --- a/web/packages/new/photos/services/search/worker.ts +++ b/web/packages/new/photos/services/search/worker.ts @@ -12,6 +12,7 @@ import * as chrono from "chrono-node"; import { expose } from "comlink"; import type { City, + DateSearchResult, Location, LocationTagData, SearchDateComponents, @@ -80,11 +81,6 @@ const dateSuggestion = ( label, })); -interface DateSearchResult { - components: SearchDateComponents; - label: string; -} - /** * Try to parse an arbitrary search string into sets of date components. * @@ -99,7 +95,7 @@ interface DateSearchResult { * In addition, also return a formatted representation of the "best" guess at * the date that was intended by the search string. */ -export const parseDateComponents = ( +const parseDateComponents = ( s: string, locale: string, holidays: DateSearchResult[], @@ -137,9 +133,6 @@ const parseChrono = (s: string, locale: string): DateSearchResult[] => format.dayPeriod = "short"; } - // TODO Temp - console.log("locale", locale); - const formatter = new Intl.DateTimeFormat(locale, format); const label = formatter.format(p.date()); return { components, label }; From ff4b388877853fb999e3a951642f264398a8b0f3 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 6 Sep 2024 11:33:40 +0530 Subject: [PATCH 18/42] Clean --- .../new/photos/services/search/worker.ts | 60 +++++++++---------- 1 file changed, 27 insertions(+), 33 deletions(-) diff --git a/web/packages/new/photos/services/search/worker.ts b/web/packages/new/photos/services/search/worker.ts index 032024c632..50ab41a870 100644 --- a/web/packages/new/photos/services/search/worker.ts +++ b/web/packages/new/photos/services/search/worker.ts @@ -155,7 +155,7 @@ const parseYearComponents = (s: string): DateSearchResult[] => { const parseHolidayComponents = (s: string, holidays: DateSearchResult[]) => holidays.filter(({ label }) => label.toLowerCase().includes(s)); -function isMatch(file: EnteFile, query: SearchQuery) { +const isMatch = (file: EnteFile, query: SearchQuery) => { if (query?.collection) { return query.collection === file.collectionID; } @@ -197,7 +197,7 @@ function isMatch(file: EnteFile, query: SearchQuery) { return query.clip.has(file.id); } return false; -} +}; const isDateComponentsMatch = ( { year, month, day, weekday, hour }: SearchDateComponents, @@ -217,47 +217,41 @@ const isDateComponentsMatch = ( return match; }; -export function isInsideLocationTag( +const defaultCityRadius = 10; +const kmsPerDegree = 111.16; + +const isInsideLocationTag = ( location: Location, locationTag: LocationTagData, -) { - return isLocationCloseToPoint( - location, - locationTag.centerPoint, - locationTag.radius, - ); -} +) => isWithinRadius(location, locationTag.centerPoint, locationTag.radius); -const DEFAULT_CITY_RADIUS = 10; -const KMS_PER_DEGREE = 111.16; - -export function isInsideCity(location: Location, city: City) { - return isLocationCloseToPoint( +const isInsideCity = (location: Location, city: City) => + isWithinRadius( { latitude: city.lat, longitude: city.lng }, location, - DEFAULT_CITY_RADIUS, + defaultCityRadius, ); -} -function isLocationCloseToPoint( +const isWithinRadius = ( centerPoint: Location, location: Location, radius: number, -) { - const a = (radius * _scaleFactor(centerPoint.latitude!)) / KMS_PER_DEGREE; - const b = radius / KMS_PER_DEGREE; +) => { + const a = + (radius * radiusScaleFactor(centerPoint.latitude!)) / kmsPerDegree; + const b = radius / kmsPerDegree; const x = centerPoint.latitude! - location.latitude!; const y = centerPoint.longitude! - location.longitude!; - if ((x * x) / (a * a) + (y * y) / (b * b) <= 1) { - return true; - } - return false; -} + return (x * x) / (a * a) + (y * y) / (b * b) <= 1; +}; -///The area bounded by the location tag becomes more elliptical with increase -///in the magnitude of the latitude on the caritesian plane. When latitude is -///0 degrees, the ellipse is a circle with a = b = r. When latitude incrases, -///the major axis (a) has to be scaled by the secant of the latitude. -function _scaleFactor(lat: number) { - return 1 / Math.cos(lat * (Math.PI / 180)); -} +/** + * A latitude specific scaling factor to apply to the radius of a location + * search. + * + * The area bounded by the location tag becomes more elliptical with increase in + * the magnitude of the latitude on the caritesian plane. When latitude is 0 + * degrees, the ellipse is a circle with a = b = r. When latitude incrases, the + * major axis (a) has to be scaled by the secant of the latitude. + */ +const radiusScaleFactor = (lat: number) => 1 / Math.cos(lat * (Math.PI / 180)); From 9e48010ee6bf3dadf82228c6eefdd304e9082ef9 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 6 Sep 2024 11:57:56 +0530 Subject: [PATCH 19/42] Latlng --- web/packages/base/types/index.ts | 7 ++++++ web/packages/media/file-metadata.ts | 23 ++++++++++++++++++- .../new/photos/services/search/types.ts | 4 ++-- .../new/photos/services/search/worker.ts | 15 +++++++----- 4 files changed, 40 insertions(+), 9 deletions(-) create mode 100644 web/packages/base/types/index.ts diff --git a/web/packages/base/types/index.ts b/web/packages/base/types/index.ts new file mode 100644 index 0000000000..10dc83fa67 --- /dev/null +++ b/web/packages/base/types/index.ts @@ -0,0 +1,7 @@ +/** + * A location, represented as a (latitude, longitude) pair. + */ +export interface Location { + latitude: number; + longitude: number; +} diff --git a/web/packages/media/file-metadata.ts b/web/packages/media/file-metadata.ts index 011200ff95..2e0d95d430 100644 --- a/web/packages/media/file-metadata.ts +++ b/web/packages/media/file-metadata.ts @@ -1,12 +1,14 @@ import { decryptMetadataJSON, encryptMetadataJSON } from "@/base/crypto"; import { authenticatedRequestHeaders, ensureOk } from "@/base/http"; import { apiURL } from "@/base/origins"; +import type { Location } from "@/base/types"; import { type EnteFile, type FilePublicMagicMetadata, } from "@/new/photos/types/file"; import { mergeMetadata1 } from "@/new/photos/utils/file"; import { ensure } from "@/utils/ensure"; +import { nullToUndefined } from "@/utils/transform"; import { z } from "zod"; import { FileType } from "./file-type"; @@ -559,7 +561,7 @@ export interface ParsedMetadata { */ creationDate?: ParsedMetadataDate; /** The GPS coordinates where the photo was taken. */ - location?: { latitude: number; longitude: number }; + location?: Location; } /** @@ -760,3 +762,22 @@ export const toUIDate = (dateLike: ParsedMetadataDate | string | number) => { return new Date(dateLike / 1000); } }; + +/** + * Return the GPS coordinates (if any) present in the given {@link EnteFile}. + */ +export const fileLocation = (enteFile: EnteFile): Location | undefined => { + // TODO: EnteFile types. Need to verify that metadata itself, and + // metadata.lat/lng can not be null (I think they likely can, if so need to + // update the types). Need to supress the linter meanwhile. + + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + if (!enteFile.metadata) return undefined; + + const latitude = nullToUndefined(enteFile.metadata.latitude); + const longitude = nullToUndefined(enteFile.metadata.longitude); + + if (latitude === undefined || longitude === undefined) return undefined; + + return { latitude, longitude }; +}; diff --git a/web/packages/new/photos/services/search/types.ts b/web/packages/new/photos/services/search/types.ts index a80799ab0e..ab1ee935e4 100644 --- a/web/packages/new/photos/services/search/types.ts +++ b/web/packages/new/photos/services/search/types.ts @@ -58,7 +58,7 @@ export interface SearchPerson { // TODO-cgroup: Audit below -export interface Location { +export interface LocationOld { latitude: number | null; longitude: number | null; } @@ -68,7 +68,7 @@ export interface LocationTagData { radius: number; aSquare: number; bSquare: number; - centerPoint: Location; + centerPoint: LocationOld; } export interface City { diff --git a/web/packages/new/photos/services/search/worker.ts b/web/packages/new/photos/services/search/worker.ts index 50ab41a870..7e0800ae36 100644 --- a/web/packages/new/photos/services/search/worker.ts +++ b/web/packages/new/photos/services/search/worker.ts @@ -13,7 +13,7 @@ import { expose } from "comlink"; import type { City, DateSearchResult, - Location, + LocationOld, LocationTagData, SearchDateComponents, SearchQuery, @@ -166,10 +166,13 @@ const isMatch = (file: EnteFile, query: SearchQuery) => { getUICreationDate(file, getPublicMagicMetadataSync(file)), ); } + + const [latitude, longitude] = [file.metadata.latitude, file.metadata.longitude]; + if (latitude !== undefined && longitude !== undefined ) if (query?.location) { return isInsideLocationTag( { - latitude: file.metadata.latitude ?? null, + latitude: ?? null, longitude: file.metadata.longitude ?? null, }, query.location, @@ -221,11 +224,11 @@ const defaultCityRadius = 10; const kmsPerDegree = 111.16; const isInsideLocationTag = ( - location: Location, + location: LocationOld, locationTag: LocationTagData, ) => isWithinRadius(location, locationTag.centerPoint, locationTag.radius); -const isInsideCity = (location: Location, city: City) => +const isInsideCity = (location: LocationOld, city: City) => isWithinRadius( { latitude: city.lat, longitude: city.lng }, location, @@ -233,8 +236,8 @@ const isInsideCity = (location: Location, city: City) => ); const isWithinRadius = ( - centerPoint: Location, - location: Location, + centerPoint: LocationOld, + location: LocationOld, radius: number, ) => { const a = From cfea74051136c3d6662b1c1f608fa0faee7d375e Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 6 Sep 2024 12:44:25 +0530 Subject: [PATCH 20/42] Parse --- .../PhotoViewer/FileInfo/MapBox.tsx | 3 +- .../components/PhotoViewer/FileInfo/index.tsx | 26 ++---- web/apps/photos/src/services/export/index.ts | 8 +- .../photos/src/services/upload/takeout.ts | 92 ++++++++++--------- web/packages/base/location.ts | 25 +++++ web/packages/base/types/index.ts | 7 -- web/packages/media/file-metadata.ts | 10 +- 7 files changed, 90 insertions(+), 81 deletions(-) create mode 100644 web/packages/base/location.ts delete mode 100644 web/packages/base/types/index.ts diff --git a/web/apps/photos/src/components/PhotoViewer/FileInfo/MapBox.tsx b/web/apps/photos/src/components/PhotoViewer/FileInfo/MapBox.tsx index e21fc32738..07bd9dbe60 100644 --- a/web/apps/photos/src/components/PhotoViewer/FileInfo/MapBox.tsx +++ b/web/apps/photos/src/components/PhotoViewer/FileInfo/MapBox.tsx @@ -1,4 +1,5 @@ import { haveWindow } from "@/base/env"; +import { type Location } from "@/base/location"; import { styled } from "@mui/material"; import { useEffect, useRef } from "react"; import { MapButton } from "./MapButton"; @@ -29,7 +30,7 @@ const MapBoxEnableContainer = styled(MapBoxContainer)` `; interface MapBoxProps { - location: { latitude: number; longitude: number }; + location: Location; mapEnabled: boolean; openUpdateMapConfirmationDialog: () => void; } diff --git a/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx b/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx index 801fcfe38e..af28c5f427 100644 --- a/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx +++ b/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx @@ -2,9 +2,11 @@ import { EnteDrawer } from "@/base/components/EnteDrawer"; import { Titlebar } from "@/base/components/Titlebar"; import { EllipsizedTypography } from "@/base/components/Typography"; import { nameAndExtension } from "@/base/file"; +import type { Location } from "@/base/location"; import log from "@/base/log"; import type { ParsedMetadata } from "@/media/file-metadata"; import { + fileLocation, getUICreationDate, updateRemotePublicMagicMetadata, type ParsedMetadataDate, @@ -97,16 +99,9 @@ export const FileInfo: React.FC = ({ const [openRawExif, setOpenRawExif] = useState(false); const location = useMemo(() => { - if (file && file.metadata) { - if ( - (file.metadata.latitude || file.metadata.latitude === 0) && - !(file.metadata.longitude === 0 && file.metadata.latitude === 0) - ) { - return { - latitude: file.metadata.latitude, - longitude: file.metadata.longitude, - }; - } + if (file) { + const location = fileLocation(file); + if (location) return location; } return exif?.parsed?.location; }, [file, exif]); @@ -181,7 +176,7 @@ export const FileInfo: React.FC = ({ !mapEnabled || publicCollectionGalleryContext.accessedThroughSharedURL ? ( = ({ } customEndButton={ @@ -531,11 +526,8 @@ const BasicDeviceCamera: React.FC<{ parsedExif: ExifInfo }> = ({ ); }; -const getOpenStreetMapLink = (location: { - latitude: number; - longitude: number; -}) => - `https://www.openstreetmap.org/?mlat=${location.latitude}&mlon=${location.longitude}#map=15/${location.latitude}/${location.longitude}`; +const openStreetMapLink = ({ latitude, longitude }: Location) => + `https://www.openstreetmap.org/?mlat=${latitude}&mlon=${longitude}#map=15/${latitude}/${longitude}`; interface RawExifProps { open: boolean; diff --git a/web/apps/photos/src/services/export/index.ts b/web/apps/photos/src/services/export/index.ts index 4af1792bb7..5b818603a8 100644 --- a/web/apps/photos/src/services/export/index.ts +++ b/web/apps/photos/src/services/export/index.ts @@ -1,6 +1,6 @@ import { ensureElectron } from "@/base/electron"; import log from "@/base/log"; -import type { Metadata } from "@/media/file-metadata"; +import { fileLocation, type Metadata } from "@/media/file-metadata"; import { FileType } from "@/media/file-type"; import { decodeLivePhoto } from "@/media/live-photo"; import downloadManager from "@/new/photos/services/download"; @@ -1383,6 +1383,7 @@ const getGoogleLikeMetadataFile = (fileExportName: string, file: EnteFile) => { (metadata.modificationTime ?? metadata.creationTime) / 1000000, ); const captionValue: string = file?.pubMagicMetadata?.data?.caption; + const geoData = fileLocation(file); return JSON.stringify( { title: fileExportName, @@ -1395,10 +1396,7 @@ const getGoogleLikeMetadataFile = (fileExportName: string, file: EnteFile) => { timestamp: modificationTime, formatted: formatDateTimeShort(modificationTime * 1000), }, - geoData: { - latitude: metadata.latitude, - longitude: metadata.longitude, - }, + geoData, }, null, 2, diff --git a/web/apps/photos/src/services/upload/takeout.ts b/web/apps/photos/src/services/upload/takeout.ts index 3ece43a4c0..62401f9576 100644 --- a/web/apps/photos/src/services/upload/takeout.ts +++ b/web/apps/photos/src/services/upload/takeout.ts @@ -2,6 +2,7 @@ import { ensureElectron } from "@/base/electron"; import { nameAndExtension } from "@/base/file"; +import { type Location } from "@/base/location"; import log from "@/base/log"; import type { UploadItem } from "@/new/photos/services/upload/types"; import { readStream } from "@/new/photos/utils/native-stream"; @@ -16,7 +17,7 @@ import { readStream } from "@/new/photos/utils/native-stream"; export interface ParsedMetadataJSON { creationTime?: number; modificationTime?: number; - location?: { latitude: number; longitude: number }; + location?: Location; } export const MAX_FILE_NAME_LENGTH_GOOGLE_EXPORT = 46; @@ -112,53 +113,60 @@ const parseMetadataJSONText = (text: string) => { const parsedMetadataJSON: ParsedMetadataJSON = {}; - // The metadata provided by Google does not include the time zone where the - // photo was taken, it only has an epoch seconds value. - if ( - metadataJSON["photoTakenTime"] && - metadataJSON["photoTakenTime"]["timestamp"] - ) { - parsedMetadataJSON.creationTime = - metadataJSON["photoTakenTime"]["timestamp"] * 1e6; - } else if ( - metadataJSON["creationTime"] && - metadataJSON["creationTime"]["timestamp"] - ) { - parsedMetadataJSON.creationTime = - metadataJSON["creationTime"]["timestamp"] * 1e6; - } + parsedMetadataJSON.creationTime = + parseGTTimestamp(metadataJSON["photoTakenTime"]) ?? + parseGTTimestamp(metadataJSON["creationTime"]); - if ( - metadataJSON["modificationTime"] && - metadataJSON["modificationTime"]["timestamp"] - ) { - parsedMetadataJSON.modificationTime = - metadataJSON["modificationTime"]["timestamp"] * 1e6; - } + parsedMetadataJSON.modificationTime = parseGTTimestamp( + metadataJSON["modificationTime"], + ); - if ( - metadataJSON["geoData"] && - (metadataJSON["geoData"]["latitude"] !== 0.0 || - metadataJSON["geoData"]["longitude"] !== 0.0) - ) { - parsedMetadataJSON.location = { - latitude: metadataJSON["geoData"]["latitude"], - longitude: metadataJSON["geoData"]["longitude"], - }; - } else if ( - metadataJSON["geoDataExif"] && - (metadataJSON["geoDataExif"]["latitude"] !== 0.0 || - metadataJSON["geoDataExif"]["longitude"] !== 0.0) - ) { - parsedMetadataJSON.location = { - latitude: metadataJSON["geoDataExif"]["latitude"], - longitude: metadataJSON["geoDataExif"]["longitude"], - }; - } + parsedMetadataJSON.location = + parseGTLocation(metadataJSON["geoData"]) ?? + parseGTLocation(metadataJSON["geoDataExif"]); return parsedMetadataJSON; }; +/** + * Parse a nullish epoch seconds timestamp from a field in a Google Takeout + * JSON, converting it into epoch microseconds if it is found. + * + * Note that the metadata provided by Google does not include the time zone + * where the photo was taken, it only has an epoch seconds value. + */ +const parseGTTimestamp = (o: unknown) => { + if ( + o && + typeof o == "object" && + "timestamp" in o && + typeof o.timestamp == "number" + ) { + const { timestamp } = o; + if (timestamp) return timestamp * 1e6; + } + return undefined; +}; + +/** + * A custom parser (instead of parseLatLng) that retains the existing behaviour + * of ignoring (0, 0) lat lng pairs when reading Google Takeout JSONs. + */ +const parseGTLocation = (o: unknown) => { + if ( + o && + typeof o == "object" && + "latitude" in o && + typeof o.latitude == "number" && + "longitude" in o && + typeof o.longitude == "number" + ) { + const { latitude, longitude } = o; + if (latitude !== 0 || longitude !== 0) return { latitude, longitude }; + } + return undefined; +}; + /** * Return the matching entry (if any) from {@link parsedMetadataJSONMap} for the * {@link fileName} and {@link collectionID} combination. diff --git a/web/packages/base/location.ts b/web/packages/base/location.ts new file mode 100644 index 0000000000..d8fbb7fa0c --- /dev/null +++ b/web/packages/base/location.ts @@ -0,0 +1,25 @@ +import { nullToUndefined } from "@/utils/transform"; + +/** + * A location, represented as a (latitude, longitude) pair. + */ +export interface Location { + latitude: number; + longitude: number; +} + +/** + * Convert a pair of nullish latitude and longitude values into a + * {@link Location} if both of them are present. + */ +export const parseLatLng = ( + latitudeN: number | undefined | null, + longitudeN: number | undefined | null, +): Location | undefined => { + const latitude = nullToUndefined(latitudeN); + const longitude = nullToUndefined(longitudeN); + + if (latitude === undefined || longitude === undefined) return undefined; + + return { latitude, longitude }; +}; diff --git a/web/packages/base/types/index.ts b/web/packages/base/types/index.ts deleted file mode 100644 index 10dc83fa67..0000000000 --- a/web/packages/base/types/index.ts +++ /dev/null @@ -1,7 +0,0 @@ -/** - * A location, represented as a (latitude, longitude) pair. - */ -export interface Location { - latitude: number; - longitude: number; -} diff --git a/web/packages/media/file-metadata.ts b/web/packages/media/file-metadata.ts index 2e0d95d430..8fc65be286 100644 --- a/web/packages/media/file-metadata.ts +++ b/web/packages/media/file-metadata.ts @@ -1,14 +1,13 @@ import { decryptMetadataJSON, encryptMetadataJSON } from "@/base/crypto"; import { authenticatedRequestHeaders, ensureOk } from "@/base/http"; +import type { Location } from "@/base/location"; import { apiURL } from "@/base/origins"; -import type { Location } from "@/base/types"; import { type EnteFile, type FilePublicMagicMetadata, } from "@/new/photos/types/file"; import { mergeMetadata1 } from "@/new/photos/utils/file"; import { ensure } from "@/utils/ensure"; -import { nullToUndefined } from "@/utils/transform"; import { z } from "zod"; import { FileType } from "./file-type"; @@ -773,11 +772,4 @@ export const fileLocation = (enteFile: EnteFile): Location | undefined => { // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition if (!enteFile.metadata) return undefined; - - const latitude = nullToUndefined(enteFile.metadata.latitude); - const longitude = nullToUndefined(enteFile.metadata.longitude); - - if (latitude === undefined || longitude === undefined) return undefined; - - return { latitude, longitude }; }; From 20247493c8770066938429a6d5c93be7a97b9566 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 6 Sep 2024 12:53:16 +0530 Subject: [PATCH 21/42] Separate parsing --- .../PhotoViewer/FileInfo/MapBox.tsx | 2 +- .../components/PhotoViewer/FileInfo/index.tsx | 2 +- web/apps/photos/src/services/export/index.ts | 33 +++++++++---------- .../photos/src/services/upload/takeout.ts | 2 +- web/packages/base/location.ts | 25 -------------- web/packages/base/types/index.ts | 7 ++++ web/packages/media/file-metadata.ts | 10 +++++- .../new/photos/services/search/worker.ts | 32 +++++++++--------- 8 files changed, 49 insertions(+), 64 deletions(-) delete mode 100644 web/packages/base/location.ts create mode 100644 web/packages/base/types/index.ts diff --git a/web/apps/photos/src/components/PhotoViewer/FileInfo/MapBox.tsx b/web/apps/photos/src/components/PhotoViewer/FileInfo/MapBox.tsx index 07bd9dbe60..0fcd6975ae 100644 --- a/web/apps/photos/src/components/PhotoViewer/FileInfo/MapBox.tsx +++ b/web/apps/photos/src/components/PhotoViewer/FileInfo/MapBox.tsx @@ -1,5 +1,5 @@ import { haveWindow } from "@/base/env"; -import { type Location } from "@/base/location"; +import { type Location } from "@/base/types"; import { styled } from "@mui/material"; import { useEffect, useRef } from "react"; import { MapButton } from "./MapButton"; diff --git a/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx b/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx index af28c5f427..7ae609c149 100644 --- a/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx +++ b/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx @@ -2,8 +2,8 @@ import { EnteDrawer } from "@/base/components/EnteDrawer"; import { Titlebar } from "@/base/components/Titlebar"; import { EllipsizedTypography } from "@/base/components/Typography"; import { nameAndExtension } from "@/base/file"; -import type { Location } from "@/base/location"; import log from "@/base/log"; +import type { Location } from "@/base/types"; import type { ParsedMetadata } from "@/media/file-metadata"; import { fileLocation, diff --git a/web/apps/photos/src/services/export/index.ts b/web/apps/photos/src/services/export/index.ts index 5b818603a8..b64eb9ae86 100644 --- a/web/apps/photos/src/services/export/index.ts +++ b/web/apps/photos/src/services/export/index.ts @@ -1382,25 +1382,22 @@ const getGoogleLikeMetadataFile = (fileExportName: string, file: EnteFile) => { const modificationTime = Math.floor( (metadata.modificationTime ?? metadata.creationTime) / 1000000, ); - const captionValue: string = file?.pubMagicMetadata?.data?.caption; - const geoData = fileLocation(file); - return JSON.stringify( - { - title: fileExportName, - caption: captionValue, - creationTime: { - timestamp: creationTime, - formatted: formatDateTimeShort(creationTime * 1000), - }, - modificationTime: { - timestamp: modificationTime, - formatted: formatDateTimeShort(modificationTime * 1000), - }, - geoData, + const result: Record = { + title: fileExportName, + creationTime: { + timestamp: creationTime, + formatted: formatDateTimeShort(creationTime * 1000), }, - null, - 2, - ); + modificationTime: { + timestamp: modificationTime, + formatted: formatDateTimeShort(modificationTime * 1000), + }, + }; + const caption = file?.pubMagicMetadata?.data?.caption; + if (caption) result.caption = caption; + const geoData = fileLocation(file); + if (geoData) result.geoData = geoData; + return JSON.stringify(result, null, 2); }; export const getMetadataFolderExportPath = (collectionExportPath: string) => diff --git a/web/apps/photos/src/services/upload/takeout.ts b/web/apps/photos/src/services/upload/takeout.ts index 62401f9576..83d9d9b0eb 100644 --- a/web/apps/photos/src/services/upload/takeout.ts +++ b/web/apps/photos/src/services/upload/takeout.ts @@ -2,8 +2,8 @@ import { ensureElectron } from "@/base/electron"; import { nameAndExtension } from "@/base/file"; -import { type Location } from "@/base/location"; import log from "@/base/log"; +import { type Location } from "@/base/types"; import type { UploadItem } from "@/new/photos/services/upload/types"; import { readStream } from "@/new/photos/utils/native-stream"; diff --git a/web/packages/base/location.ts b/web/packages/base/location.ts deleted file mode 100644 index d8fbb7fa0c..0000000000 --- a/web/packages/base/location.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { nullToUndefined } from "@/utils/transform"; - -/** - * A location, represented as a (latitude, longitude) pair. - */ -export interface Location { - latitude: number; - longitude: number; -} - -/** - * Convert a pair of nullish latitude and longitude values into a - * {@link Location} if both of them are present. - */ -export const parseLatLng = ( - latitudeN: number | undefined | null, - longitudeN: number | undefined | null, -): Location | undefined => { - const latitude = nullToUndefined(latitudeN); - const longitude = nullToUndefined(longitudeN); - - if (latitude === undefined || longitude === undefined) return undefined; - - return { latitude, longitude }; -}; diff --git a/web/packages/base/types/index.ts b/web/packages/base/types/index.ts new file mode 100644 index 0000000000..10dc83fa67 --- /dev/null +++ b/web/packages/base/types/index.ts @@ -0,0 +1,7 @@ +/** + * A location, represented as a (latitude, longitude) pair. + */ +export interface Location { + latitude: number; + longitude: number; +} diff --git a/web/packages/media/file-metadata.ts b/web/packages/media/file-metadata.ts index 8fc65be286..7741030cf5 100644 --- a/web/packages/media/file-metadata.ts +++ b/web/packages/media/file-metadata.ts @@ -1,13 +1,14 @@ import { decryptMetadataJSON, encryptMetadataJSON } from "@/base/crypto"; import { authenticatedRequestHeaders, ensureOk } from "@/base/http"; -import type { Location } from "@/base/location"; import { apiURL } from "@/base/origins"; +import { type Location } from "@/base/types"; import { type EnteFile, type FilePublicMagicMetadata, } from "@/new/photos/types/file"; import { mergeMetadata1 } from "@/new/photos/utils/file"; import { ensure } from "@/utils/ensure"; +import { nullToUndefined } from "@/utils/transform"; import { z } from "zod"; import { FileType } from "./file-type"; @@ -772,4 +773,11 @@ export const fileLocation = (enteFile: EnteFile): Location | undefined => { // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition if (!enteFile.metadata) return undefined; + + const latitude = nullToUndefined(enteFile.metadata.latitude); + const longitude = nullToUndefined(enteFile.metadata.longitude); + + if (latitude === undefined || longitude === undefined) return undefined; + + return { latitude, longitude }; }; diff --git a/web/packages/new/photos/services/search/worker.ts b/web/packages/new/photos/services/search/worker.ts index 7e0800ae36..51040c3ae7 100644 --- a/web/packages/new/photos/services/search/worker.ts +++ b/web/packages/new/photos/services/search/worker.ts @@ -2,7 +2,7 @@ /* eslint-disable @typescript-eslint/no-non-null-assertion */ /* eslint-disable @typescript-eslint/prefer-includes */ /* eslint-disable @typescript-eslint/no-unnecessary-condition */ -import { getUICreationDate } from "@/media/file-metadata"; +import { fileLocation, getUICreationDate } from "@/media/file-metadata"; import type { EnteFile } from "@/new/photos/types/file"; import { wait } from "@/utils/promise"; import { nullToUndefined } from "@/utils/transform"; @@ -167,38 +167,36 @@ const isMatch = (file: EnteFile, query: SearchQuery) => { ); } - const [latitude, longitude] = [file.metadata.latitude, file.metadata.longitude]; - if (latitude !== undefined && longitude !== undefined ) if (query?.location) { - return isInsideLocationTag( - { - latitude: ?? null, - longitude: file.metadata.longitude ?? null, - }, - query.location, - ); + const location = fileLocation(file); + if (!location) return false; + + return isInsideLocationTag(location, query.location); } + if (query?.city) { - return isInsideCity( - { - latitude: file.metadata.latitude ?? null, - longitude: file.metadata.longitude ?? null, - }, - query.city, - ); + const location = fileLocation(file); + if (!location) return false; + + return isInsideCity(location, query.city); } + if (query?.files) { return query.files.indexOf(file.id) !== -1; } + if (query?.person) { return query.person.files.indexOf(file.id) !== -1; } + if (typeof query?.fileType !== "undefined") { return query.fileType === file.metadata.fileType; } + if (typeof query?.clip !== "undefined") { return query.clip.has(file.id); } + return false; }; From e6605d7ac964fed0a6c7a9d8486831b770d1cba0 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 6 Sep 2024 13:40:55 +0530 Subject: [PATCH 22/42] Log --- web/packages/accounts/pages/verify.tsx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/web/packages/accounts/pages/verify.tsx b/web/packages/accounts/pages/verify.tsx index d2e468571f..604c36b718 100644 --- a/web/packages/accounts/pages/verify.tsx +++ b/web/packages/accounts/pages/verify.tsx @@ -42,6 +42,7 @@ import { import { configureSRP } from "../services/srp"; import type { PageProps } from "../types/page"; import type { SRPSetupAttributes } from "../types/srp"; +import log from "@/base/log"; const Page: React.FC = ({ appContext }) => { const { logout, showNavBar, setDialogBoxAttributesV2 } = appContext; @@ -159,6 +160,7 @@ const Page: React.FC = ({ appContext }) => { setFieldError(t("EXPIRED_CODE")); } } else { + log.error("OTT verification failed", e); setFieldError(`${t("UNKNOWN_ERROR")} ${JSON.stringify(e)}`); } } From 261a7f278b6160b122dfa9e01d73a2a23a252c32 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 6 Sep 2024 13:41:04 +0530 Subject: [PATCH 23/42] Fix --- web/packages/shared/storage/localStorage/index.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/web/packages/shared/storage/localStorage/index.ts b/web/packages/shared/storage/localStorage/index.ts index 6330f21d9a..156f466261 100644 --- a/web/packages/shared/storage/localStorage/index.ts +++ b/web/packages/shared/storage/localStorage/index.ts @@ -70,11 +70,13 @@ export const setLSUser = async (user: object) => { export const migrateKVToken = async (user: unknown) => { // Throw an error if the data is in local storage but not in IndexedDB. This // is a pre-cursor to inlining this code. + // TODO(REL): Remove this sanity check after a few days. + const oldLSUser = getData(LS_KEYS.USER); const wasMissing = - user && - typeof user == "object" && - "token" in user && - typeof user.token == "string" && + oldLSUser && + typeof oldLSUser == "object" && + "token" in oldLSUser && + typeof oldLSUser.token == "string" && !(await getKV("token")); user && From 55ece20d70a3d64e9106934b8fa2773a8fdd8d32 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 6 Sep 2024 14:11:58 +0530 Subject: [PATCH 24/42] timestamp is a string --- web/apps/photos/src/services/upload/takeout.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/apps/photos/src/services/upload/takeout.ts b/web/apps/photos/src/services/upload/takeout.ts index 83d9d9b0eb..d95a53808b 100644 --- a/web/apps/photos/src/services/upload/takeout.ts +++ b/web/apps/photos/src/services/upload/takeout.ts @@ -140,10 +140,10 @@ const parseGTTimestamp = (o: unknown) => { o && typeof o == "object" && "timestamp" in o && - typeof o.timestamp == "number" + typeof o.timestamp == "string" ) { const { timestamp } = o; - if (timestamp) return timestamp * 1e6; + if (timestamp) return parseInt(timestamp) * 1e6; } return undefined; }; From b742ffcafd353619b86af9817ce7be5148201a72 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 6 Sep 2024 14:34:46 +0530 Subject: [PATCH 25/42] Doc --- .../photos/src/services/upload/takeout.ts | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/web/apps/photos/src/services/upload/takeout.ts b/web/apps/photos/src/services/upload/takeout.ts index d95a53808b..de0f475b65 100644 --- a/web/apps/photos/src/services/upload/takeout.ts +++ b/web/apps/photos/src/services/upload/takeout.ts @@ -129,13 +129,15 @@ const parseMetadataJSONText = (text: string) => { }; /** - * Parse a nullish epoch seconds timestamp from a field in a Google Takeout - * JSON, converting it into epoch microseconds if it is found. + * Parse a nullish epoch seconds timestamp string from a field in a Google + * Takeout JSON, converting it into epoch microseconds if it is found. * * Note that the metadata provided by Google does not include the time zone - * where the photo was taken, it only has an epoch seconds value. + * where the photo was taken, it only has an epoch seconds value. There is an + * associated formatted date value (e.g. "17 Feb 2021, 03:22:16 UTC") but that + * seems to be in UTC and doesn't have the time zone either. */ -const parseGTTimestamp = (o: unknown) => { +const parseGTTimestamp = (o: unknown): number | undefined => { if ( o && typeof o == "object" && @@ -143,16 +145,18 @@ const parseGTTimestamp = (o: unknown) => { typeof o.timestamp == "string" ) { const { timestamp } = o; - if (timestamp) return parseInt(timestamp) * 1e6; + if (timestamp) return parseInt(timestamp, 10) * 1e6; } return undefined; }; /** - * A custom parser (instead of parseLatLng) that retains the existing behaviour - * of ignoring (0, 0) lat lng pairs when reading Google Takeout JSONs. + * Parse a (latitude, longitude) location pair field in a Google Takeout JSON. + * + * Apparently Google puts in (0, 0) to indicate missing data, so this function + * only returns a parsed result if both components are present and non-zero. */ -const parseGTLocation = (o: unknown) => { +const parseGTLocation = (o: unknown): Location | undefined => { if ( o && typeof o == "object" && From 49a81c10db0cf11a4fcd6cabfe97190f5e170ee7 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 6 Sep 2024 15:48:16 +0530 Subject: [PATCH 26/42] Better mirror the google format by including seconds and using UTC --- web/apps/photos/src/services/export/index.ts | 49 +++++++++++++++++--- web/packages/shared/time/format.ts | 11 ----- 2 files changed, 43 insertions(+), 17 deletions(-) diff --git a/web/apps/photos/src/services/export/index.ts b/web/apps/photos/src/services/export/index.ts index b64eb9ae86..261d3d5e33 100644 --- a/web/apps/photos/src/services/export/index.ts +++ b/web/apps/photos/src/services/export/index.ts @@ -1,6 +1,10 @@ import { ensureElectron } from "@/base/electron"; import log from "@/base/log"; -import { fileLocation, type Metadata } from "@/media/file-metadata"; +import { + fileLocation, + getUICreationDate, + type Metadata, +} from "@/media/file-metadata"; import { FileType } from "@/media/file-type"; import { decodeLivePhoto } from "@/media/live-photo"; import downloadManager from "@/new/photos/services/download"; @@ -17,12 +21,12 @@ import { writeStream } from "@/new/photos/utils/native-stream"; import { wait } from "@/utils/promise"; import { CustomError } from "@ente/shared/error"; import { LS_KEYS, getData, setData } from "@ente/shared/storage/localStorage"; -import { formatDateTimeShort } from "@ente/shared/time/format"; import type { User } from "@ente/shared/user/types"; import QueueProcessor, { CancellationStatus, RequestCanceller, } from "@ente/shared/utils/queueProcessor"; +import i18n from "i18next"; import { Collection } from "types/collection"; import { CollectionExportNames, @@ -100,6 +104,7 @@ class ExportService { success: 0, failed: 0, }; + private cachedMetadataDateTimeFormatter: Intl.DateTimeFormat; getExportSettings(): ExportSettings { try { @@ -1076,12 +1081,38 @@ class ExportService { fileExportName: string, file: EnteFile, ) { + const formatter = this.metadataDateTimeFormatter(); await ensureElectron().fs.writeFile( getFileMetadataExportPath(collectionExportPath, fileExportName), - getGoogleLikeMetadataFile(fileExportName, file), + getGoogleLikeMetadataFile(fileExportName, file, formatter), ); } + /** + * Lazily created, cached instance of the date time formatter that should be + * used for formatting the dates added to the metadata file. + */ + private metadataDateTimeFormatter() { + if (this.cachedMetadataDateTimeFormatter) + return this.cachedMetadataDateTimeFormatter; + + // AFAIK, Google's format is not documented. It also seems to vary with + // locale. This is a best attempt at constructing a formatter that + // mirrors the format used by the timestamps in the takeout JSON. + const formatter = new Intl.DateTimeFormat(i18n.language, { + month: "short", + day: "numeric", + year: "numeric", + hour: "numeric", + minute: "numeric", + second: "numeric", + timeZoneName: "short", + timeZone: "UTC", + }); + this.cachedMetadataDateTimeFormatter = formatter; + return formatter; + } + isExportInProgress = () => { return this.exportInProgress; }; @@ -1376,7 +1407,11 @@ const getCollectionExportedFiles = ( return collectionExportedFiles; }; -const getGoogleLikeMetadataFile = (fileExportName: string, file: EnteFile) => { +const getGoogleLikeMetadataFile = ( + fileExportName: string, + file: EnteFile, + dateTimeFormatter: Intl.DateTimeFormat, +) => { const metadata: Metadata = file.metadata; const creationTime = Math.floor(metadata.creationTime / 1000000); const modificationTime = Math.floor( @@ -1386,11 +1421,13 @@ const getGoogleLikeMetadataFile = (fileExportName: string, file: EnteFile) => { title: fileExportName, creationTime: { timestamp: creationTime, - formatted: formatDateTimeShort(creationTime * 1000), + formatted: dateTimeFormatter.format( + getUICreationDate(file, file.pubMagicMetadata?.data), + ), }, modificationTime: { timestamp: modificationTime, - formatted: formatDateTimeShort(modificationTime * 1000), + formatted: dateTimeFormatter.format(modificationTime * 1000), }, }; const caption = file?.pubMagicMetadata?.data?.caption; diff --git a/web/packages/shared/time/format.ts b/web/packages/shared/time/format.ts index 47a187093b..9caa5daab9 100644 --- a/web/packages/shared/time/format.ts +++ b/web/packages/shared/time/format.ts @@ -9,13 +9,6 @@ const dateTimeFullFormatter1 = new Intl.DateTimeFormat(i18n.language, { const dateTimeFullFormatter2 = new Intl.DateTimeFormat(i18n.language, { year: "numeric", }); -const dateTimeShortFormatter = new Intl.DateTimeFormat(i18n.language, { - month: "short", - day: "numeric", - year: "numeric", - hour: "2-digit", - minute: "2-digit", -}); const timeFormatter = new Intl.DateTimeFormat(i18n.language, { timeStyle: "short", @@ -37,10 +30,6 @@ export function formatDate(date: number | Date) { .join(" "); } -export function formatDateTimeShort(date: number | Date) { - return dateTimeShortFormatter.format(date); -} - export function formatTime(date: number | Date) { return timeFormatter.format(date).toUpperCase(); } From 85397732c81bc569a93d26f97d50e46e882c0e26 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 6 Sep 2024 16:12:41 +0530 Subject: [PATCH 27/42] Rename --- .../photos/src/components/FixCreationTime.tsx | 6 +++--- .../components/PhotoViewer/FileInfo/index.tsx | 4 ++-- web/apps/photos/src/services/export/index.ts | 4 ++-- web/packages/media/file-metadata.ts | 19 +++++++++++-------- .../new/photos/services/search/worker.ts | 4 ++-- 5 files changed, 20 insertions(+), 17 deletions(-) diff --git a/web/apps/photos/src/components/FixCreationTime.tsx b/web/apps/photos/src/components/FixCreationTime.tsx index d427b52ad5..4735aa7722 100644 --- a/web/apps/photos/src/components/FixCreationTime.tsx +++ b/web/apps/photos/src/components/FixCreationTime.tsx @@ -1,7 +1,7 @@ import log from "@/base/log"; import { decryptPublicMagicMetadata, - getUICreationDate, + fileCreationPhotoDate, updateRemotePublicMagicMetadata, type ParsedMetadataDate, } from "@/media/file-metadata"; @@ -352,11 +352,11 @@ const updateEnteFileDate = async ( if (!newDate) return; - const existingUIDate = getUICreationDate( + const existingDate = fileCreationPhotoDate( enteFile, await decryptPublicMagicMetadata(enteFile), ); - if (newDate.timestamp == existingUIDate.getTime()) return; + if (newDate.timestamp == existingDate.getTime()) return; await updateRemotePublicMagicMetadata(enteFile, { dateTime: newDate.dateTime, diff --git a/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx b/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx index 7ae609c149..f36e87b7dd 100644 --- a/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx +++ b/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx @@ -6,8 +6,8 @@ import log from "@/base/log"; import type { Location } from "@/base/types"; import type { ParsedMetadata } from "@/media/file-metadata"; import { + fileCreationPhotoDate, fileLocation, - getUICreationDate, updateRemotePublicMagicMetadata, type ParsedMetadataDate, } from "@/media/file-metadata"; @@ -362,7 +362,7 @@ export const CreationTime: React.FC = ({ const closeEditMode = () => setIsInEditMode(false); const publicMagicMetadata = getPublicMagicMetadataSync(enteFile); - const originalDate = getUICreationDate(enteFile, publicMagicMetadata); + const originalDate = fileCreationPhotoDate(enteFile, publicMagicMetadata); const saveEdits = async (pickedTime: ParsedMetadataDate) => { try { diff --git a/web/apps/photos/src/services/export/index.ts b/web/apps/photos/src/services/export/index.ts index 261d3d5e33..c65b7ceadc 100644 --- a/web/apps/photos/src/services/export/index.ts +++ b/web/apps/photos/src/services/export/index.ts @@ -1,8 +1,8 @@ import { ensureElectron } from "@/base/electron"; import log from "@/base/log"; import { + fileCreationPhotoDate, fileLocation, - getUICreationDate, type Metadata, } from "@/media/file-metadata"; import { FileType } from "@/media/file-type"; @@ -1422,7 +1422,7 @@ const getGoogleLikeMetadataFile = ( creationTime: { timestamp: creationTime, formatted: dateTimeFormatter.format( - getUICreationDate(file, file.pubMagicMetadata?.data), + fileCreationPhotoDate(file, file.pubMagicMetadata?.data), ), }, modificationTime: { diff --git a/web/packages/media/file-metadata.ts b/web/packages/media/file-metadata.ts index 7741030cf5..507f023c71 100644 --- a/web/packages/media/file-metadata.ts +++ b/web/packages/media/file-metadata.ts @@ -329,15 +329,16 @@ const withoutNullAndUndefinedValues = (o: object) => ); /** - * Return the file's creation date in a form suitable for using in the UI. + * Return the file's creation date as a Date in the hypothetical "timezone of + * the photo". * - * For all the details and nuance, see {@link toUIDate}. + * For all the details and nuance, see {@link createPhotoDate}. */ -export const getUICreationDate = ( +export const fileCreationPhotoDate = ( enteFile: EnteFile, publicMagicMetadata: PublicMagicMetadata | undefined, ) => - toUIDate( + createPhotoDate( publicMagicMetadata?.dateTime ?? publicMagicMetadata?.editedTime ?? enteFile.metadata.creationTime, @@ -726,9 +727,9 @@ export const parseMetadataDate = ( const dropLast = (s: string) => (s ? s.substring(0, s.length - 1) : s); /** - * Return a date that can be used on the UI by constructing it from a - * {@link ParsedMetadataDate}, or its {@link dateTime} component, or a UTC epoch - * timestamp. + * Return a date that can be used on the represent a photo on the UI, by + * constructing it from a {@link ParsedMetadataDate}, or its {@link dateTime} + * component, or a UTC epoch timestamp. * * These dates are all hypothetically in the timezone of the place where the * photo was taken. Different photos might've been taken in different timezones, @@ -747,7 +748,9 @@ const dropLast = (s: string) => (s ? s.substring(0, s.length - 1) : s); * * See also: [Note: Photos are always in local date/time]. */ -export const toUIDate = (dateLike: ParsedMetadataDate | string | number) => { +export const createPhotoDate = ( + dateLike: ParsedMetadataDate | string | number, +) => { switch (typeof dateLike) { case "object": // A ISO 8601 string without a timezone. The Date constructor will diff --git a/web/packages/new/photos/services/search/worker.ts b/web/packages/new/photos/services/search/worker.ts index 51040c3ae7..cdfea0b22d 100644 --- a/web/packages/new/photos/services/search/worker.ts +++ b/web/packages/new/photos/services/search/worker.ts @@ -2,7 +2,7 @@ /* eslint-disable @typescript-eslint/no-non-null-assertion */ /* eslint-disable @typescript-eslint/prefer-includes */ /* eslint-disable @typescript-eslint/no-unnecessary-condition */ -import { fileLocation, getUICreationDate } from "@/media/file-metadata"; +import { fileCreationPhotoDate, fileLocation } from "@/media/file-metadata"; import type { EnteFile } from "@/new/photos/types/file"; import { wait } from "@/utils/promise"; import { nullToUndefined } from "@/utils/transform"; @@ -163,7 +163,7 @@ const isMatch = (file: EnteFile, query: SearchQuery) => { if (query?.date) { return isDateComponentsMatch( query.date, - getUICreationDate(file, getPublicMagicMetadataSync(file)), + fileCreationPhotoDate(file, getPublicMagicMetadataSync(file)), ); } From 77ac215b76e218d5fd7a7eef1d7d270d0dda625a Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 6 Sep 2024 16:18:28 +0530 Subject: [PATCH 28/42] Fix lint --- web/packages/new/photos/services/search/worker.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/web/packages/new/photos/services/search/worker.ts b/web/packages/new/photos/services/search/worker.ts index cdfea0b22d..7edbcd3272 100644 --- a/web/packages/new/photos/services/search/worker.ts +++ b/web/packages/new/photos/services/search/worker.ts @@ -1,6 +1,5 @@ // TODO-cgroups /* eslint-disable @typescript-eslint/no-non-null-assertion */ -/* eslint-disable @typescript-eslint/prefer-includes */ /* eslint-disable @typescript-eslint/no-unnecessary-condition */ import { fileCreationPhotoDate, fileLocation } from "@/media/file-metadata"; import type { EnteFile } from "@/new/photos/types/file"; @@ -182,11 +181,11 @@ const isMatch = (file: EnteFile, query: SearchQuery) => { } if (query?.files) { - return query.files.indexOf(file.id) !== -1; + return query.files.includes(file.id); } if (query?.person) { - return query.person.files.indexOf(file.id) !== -1; + return query.person.files.includes(file.id); } if (typeof query?.fileType !== "undefined") { From fc9506942101aaf2668610e982549bfdc496d72a Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 6 Sep 2024 16:52:28 +0530 Subject: [PATCH 29/42] City parse 1 --- .../src/services/locationSearchService.ts | 2 +- web/apps/photos/src/services/searchService.ts | 4 +- .../new/photos/services/search/types.ts | 25 ++++++-- .../new/photos/services/search/worker.ts | 62 ++++++++++++++++--- 4 files changed, 76 insertions(+), 17 deletions(-) diff --git a/web/apps/photos/src/services/locationSearchService.ts b/web/apps/photos/src/services/locationSearchService.ts index 9e3ff032d3..66454965e3 100644 --- a/web/apps/photos/src/services/locationSearchService.ts +++ b/web/apps/photos/src/services/locationSearchService.ts @@ -31,7 +31,7 @@ class LocationSearchService { } await this.citiesPromise; return this.cities.filter((city) => { - return city.city + return city.name .toLowerCase() .startsWith(searchTerm.toLowerCase()); }); diff --git a/web/apps/photos/src/services/searchService.ts b/web/apps/photos/src/services/searchService.ts index 943edbbb47..259dc18e48 100644 --- a/web/apps/photos/src/services/searchService.ts +++ b/web/apps/photos/src/services/searchService.ts @@ -220,7 +220,7 @@ async function getLocationSuggestions(searchPhrase: string) { await locationSearchService.searchCities(searchPhrase); const nonConflictingCityResult = citySearchResults.filter( - (city) => !locationTagNames.has(city.city), + (city) => !locationTagNames.has(city.name), ); const citySearchSuggestions = nonConflictingCityResult.map( @@ -228,7 +228,7 @@ async function getLocationSuggestions(searchPhrase: string) { ({ type: SuggestionType.CITY, value: city, - label: city.city, + label: city.name, }) as Suggestion, ); diff --git a/web/packages/new/photos/services/search/types.ts b/web/packages/new/photos/services/search/types.ts index ab1ee935e4..5841899ab7 100644 --- a/web/packages/new/photos/services/search/types.ts +++ b/web/packages/new/photos/services/search/types.ts @@ -3,6 +3,7 @@ * and the search worker that does the actual searching (`worker.ts`). */ +import type { Location } from "@/base/types"; import { FileType } from "@/media/file-type"; import type { MLStatus } from "@/new/photos/services/ml"; import type { EnteFile } from "@/new/photos/types/file"; @@ -71,12 +72,24 @@ export interface LocationTagData { centerPoint: LocationOld; } -export interface City { - city: string; - country: string; - lat: number; - lng: number; -} +/** + * A city as identified by a static dataset. + * + * Each city is represented by its latitude and longitude. The dataset does not + * have information about the city's estimated radius. + */ +export type City = Location & { + /** + * Name of the city. + */ + name: string; + /** + * Name of the city, lowercased. + * + * Precomputing this save an lowercasing during the search itself. + */ + lowercasedName: string; +}; export enum SuggestionType { DATE = "DATE", diff --git a/web/packages/new/photos/services/search/worker.ts b/web/packages/new/photos/services/search/worker.ts index 7edbcd3272..d2fb6e40bf 100644 --- a/web/packages/new/photos/services/search/worker.ts +++ b/web/packages/new/photos/services/search/worker.ts @@ -1,6 +1,7 @@ // TODO-cgroups /* eslint-disable @typescript-eslint/no-non-null-assertion */ /* eslint-disable @typescript-eslint/no-unnecessary-condition */ +import { HTTPError } from "@/base/http"; import { fileCreationPhotoDate, fileLocation } from "@/media/file-metadata"; import type { EnteFile } from "@/new/photos/types/file"; import { wait } from "@/utils/promise"; @@ -9,6 +10,7 @@ import { getPublicMagicMetadataSync } from "@ente/shared/file-metadata"; import type { Component } from "chrono-node"; import * as chrono from "chrono-node"; import { expose } from "comlink"; +import { z } from "zod"; import type { City, DateSearchResult, @@ -26,6 +28,8 @@ import { SuggestionType } from "./types"; */ export class SearchWorker { private enteFiles: EnteFile[] = []; + private cities: City[] = []; + private citiesPromise: Promise | undefined; /** * Set the files that we should search across. @@ -42,7 +46,18 @@ export class SearchWorker { locale: string, holidays: DateSearchResult[], ) { - return createSearchQuery(searchString, locale, holidays); + this.triggerCityFetchIfNeeded(); + return createSearchQuery(searchString, locale, holidays, this.cities); + } + + /** + * Lazily trigger a fetch of city data, but don't wait for it to complete. + */ + triggerCityFetchIfNeeded() { + if (this.citiesPromise) return; + this.citiesPromise = fetchCities().then((cs) => { + this.cities = cs; + }); } /** @@ -59,6 +74,7 @@ const createSearchQuery = async ( searchString: string, locale: string, holidays: DateSearchResult[], + cities: City[], ): Promise => { // Normalize it by trimming whitespace and converting to lowercase. const s = searchString.trim().toLowerCase(); @@ -66,10 +82,10 @@ const createSearchQuery = async ( // TODO Temp await wait(0); - return [dateSuggestion(s, locale, holidays)].flat(); + return [dateSuggestions(s, locale, holidays)].flat(); }; -const dateSuggestion = ( +const dateSuggestions = ( s: string, locale: string, holidays: DateSearchResult[], @@ -154,6 +170,40 @@ const parseYearComponents = (s: string): DateSearchResult[] => { const parseHolidayComponents = (s: string, holidays: DateSearchResult[]) => holidays.filter(({ label }) => label.toLowerCase().includes(s)); +/** + * Zod schema describing world_cities.json. + * + * The entries also have a country field which we don't currently use. + */ +const RemoteWorldCities = z.object({ + data: z.array( + z.object({ + city: z.string(), + lat: z.number(), + lng: z.number(), + }), + ), +}); + +const fetchCities = async () => { + const res = await fetch("https://static.ente.io/world_cities.json"); + if (!res.ok) throw new HTTPError(res); + return RemoteWorldCities.parse(await res.json()).data.map( + ({ city, lat, lng }) => ({ + name: city, + lowercasedName: city.toLowerCase(), + latitude: lat, + longitude: lng, + }), + ); +}; + +/** + * Return all cities whose name begins with the given search string. + */ +const matchingCities = (s: string, cities: City[]) => + cities.filter(({ lowercasedName }) => lowercasedName.startsWith(s)); + const isMatch = (file: EnteFile, query: SearchQuery) => { if (query?.collection) { return query.collection === file.collectionID; @@ -226,11 +276,7 @@ const isInsideLocationTag = ( ) => isWithinRadius(location, locationTag.centerPoint, locationTag.radius); const isInsideCity = (location: LocationOld, city: City) => - isWithinRadius( - { latitude: city.lat, longitude: city.lng }, - location, - defaultCityRadius, - ); + isWithinRadius(city, location, defaultCityRadius); const isWithinRadius = ( centerPoint: LocationOld, From fce4295e2a52d765f599806f8c554152e74d8af6 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 6 Sep 2024 18:08:38 +0530 Subject: [PATCH 30/42] PF --- web/apps/photos/src/pages/gallery/index.tsx | 1 - web/apps/photos/src/services/sync.ts | 2 ++ web/packages/new/photos/services/search/index.ts | 9 +++++++++ web/packages/new/photos/services/search/worker.ts | 9 ++++++++- 4 files changed, 19 insertions(+), 2 deletions(-) diff --git a/web/apps/photos/src/pages/gallery/index.tsx b/web/apps/photos/src/pages/gallery/index.tsx index cd524d3f1e..0aa578a5c7 100644 --- a/web/apps/photos/src/pages/gallery/index.tsx +++ b/web/apps/photos/src/pages/gallery/index.tsx @@ -387,7 +387,6 @@ export default function Gallery() { setIsFirstLoad(false); setJustSignedUp(false); setIsFirstFetch(false); - locationSearchService.loadCities(); syncInterval.current = setInterval(() => { syncWithRemote(false, true); }, SYNC_INTERVAL_IN_MICROSECONDS); diff --git a/web/apps/photos/src/services/sync.ts b/web/apps/photos/src/services/sync.ts index c42f55d89f..ce8defd900 100644 --- a/web/apps/photos/src/services/sync.ts +++ b/web/apps/photos/src/services/sync.ts @@ -4,6 +4,7 @@ import { triggerMLStatusSync, triggerMLSync, } from "@/new/photos/services/ml"; +import { triggerSearchDataPrefetchIfNeeded } from "@/new/photos/services/search"; import { syncLocationTags } from "@/new/photos/services/user-entity"; import { syncEntities } from "services/entityService"; import { syncMapEnabled } from "services/userService"; @@ -42,4 +43,5 @@ export const sync = async () => { syncMapEnabled(), ]); if (isMLSupported) triggerMLSync(); + triggerSearchDataPrefetchIfNeeded(); }; diff --git a/web/packages/new/photos/services/search/index.ts b/web/packages/new/photos/services/search/index.ts index 6c6dcb5ee6..de3dc0add8 100644 --- a/web/packages/new/photos/services/search/index.ts +++ b/web/packages/new/photos/services/search/index.ts @@ -24,6 +24,15 @@ const createComlinkWorker = () => new Worker(new URL("worker.ts", import.meta.url)), ); +/** + * Preload any data that would be needed if the user were to search. + * + * This is an optimization to try and ensure we always have the latest state + * by the time the user gets around to searching. + */ +export const triggerSearchDataPrefetchIfNeeded = () => + void worker().then((w) => w.prefetchIfNeeded()); + /** * Set the files over which we will search. */ diff --git a/web/packages/new/photos/services/search/worker.ts b/web/packages/new/photos/services/search/worker.ts index d2fb6e40bf..031493570e 100644 --- a/web/packages/new/photos/services/search/worker.ts +++ b/web/packages/new/photos/services/search/worker.ts @@ -31,6 +31,13 @@ export class SearchWorker { private cities: City[] = []; private citiesPromise: Promise | undefined; + /** + * Prefetch any data that we might need when the actual search happens. + */ + prefetchIfNeeded() { + this.triggerCityFetchIfNeeded(); + } + /** * Set the files that we should search across. */ @@ -46,7 +53,7 @@ export class SearchWorker { locale: string, holidays: DateSearchResult[], ) { - this.triggerCityFetchIfNeeded(); + this.prefetchIfNeeded(); return createSearchQuery(searchString, locale, holidays, this.cities); } From bf7cbe141dab4320a425c3654512332f3b7bcb05 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 6 Sep 2024 18:42:14 +0530 Subject: [PATCH 31/42] lt wip --- .../new/photos/services/migrations.ts | 14 +++- .../new/photos/services/user-entity.ts | 70 ++++++++++--------- 2 files changed, 49 insertions(+), 35 deletions(-) diff --git a/web/packages/new/photos/services/migrations.ts b/web/packages/new/photos/services/migrations.ts index 6123f0c879..c4b0df2183 100644 --- a/web/packages/new/photos/services/migrations.ts +++ b/web/packages/new/photos/services/migrations.ts @@ -29,10 +29,11 @@ import { deleteDB } from "idb"; */ export const runMigrations = async () => { const m = (await getKVN("migrationLevel")) ?? 0; - const latest = 1; + const latest = 2; if (m < latest) { log.info(`Running migrations ${m} => ${latest}`); if (m < 1 && isDesktop) await m0(); + if (m < 2) await m1(); await setKV("migrationLevel", latest); } }; @@ -64,3 +65,14 @@ const m0 = () => // Delete legacy ML keys. localStorage.removeItem("faceIndexingEnabled"); }); + +// Added: Sep 2024 (v1.7.5-beta). Prunable. +const m1 = () => + // Older versions of the user-entities code kept the diff related state + // in a different place. These entries are not needed anymore (the tags + // themselves will get resynced). + Promise.allSettled([ + localForage.removeItem("location_tags"), + localForage.removeItem("location_tags_key"), + localForage.removeItem("location_tags_time"), + ]); diff --git a/web/packages/new/photos/services/user-entity.ts b/web/packages/new/photos/services/user-entity.ts index 1493d4efaf..f433449102 100644 --- a/web/packages/new/photos/services/user-entity.ts +++ b/web/packages/new/photos/services/user-entity.ts @@ -9,9 +9,7 @@ import { getKV, getKVN, setKV } from "@/base/kv"; import { apiURL } from "@/base/origins"; import { masterKeyFromSession } from "@/base/session-store"; import { ensure } from "@/utils/ensure"; -import { wait } from "@/utils/promise"; import { nullToUndefined } from "@/utils/transform"; -import localForage from "@ente/shared/storage/localForage"; import { z } from "zod"; import { gunzip } from "./gzip"; import type { CGroup } from "./ml/cgroups"; @@ -37,11 +35,6 @@ export type EntityType = */ | "cgroup"; -interface LocationTag { - id: string; - name: string; -} - /** * Sync our local location tags with those on remote. * @@ -51,19 +44,21 @@ interface LocationTag { */ export const syncLocationTags = async () => { const decoder = new TextDecoder(); - const parse = (id: string, data: Uint8Array): LocationTag => { - const rl = RemoteLocation.parse(JSON.parse(decoder.decode(data))); - return { - id, - name: rl.name, - }; - }; + const parse = (id: string, data: Uint8Array): LocationTag => ({ + id, + ...RemoteLocationTag.parse(JSON.parse(decoder.decode(data))), + }); const processBatch = async (entities: UserEntityChange[]) => { - console.log( - entities.map(({ id, data }) => (data ? parse(id, data) : id)), + const existingTagsByID = new Map( + (await savedLocationTags()).map((t) => [t.id, t]), ); - await wait(0); + entities.forEach(({ id, data }) => + data + ? existingTagsByID.set(id, parse(id, data)) + : existingTagsByID.delete(id), + ); + return saveLocationTags([...existingTagsByID.values()]); }; // TODO-cgroup: Call me @@ -71,31 +66,38 @@ export const syncLocationTags = async () => { return syncUserEntity("location", processBatch); }; -// TODO-cgroup: Call me -export const removeLegacyDBState = async () => { - // Older versions of the code kept the diff related state in a different - // place. These entries are not needed anymore. - // - // This code was added Sep 2024 and can be removed soon after a few builds - // have gone out (tag: Migration). - await Promise.allSettled([ - localForage.removeItem("location_tags"), - localForage.removeItem("location_tags_key"), - localForage.removeItem("location_tags_time"), - ]); -}; - -const RemoteLocation = z.object({ +/** Zod schema for the tag that we get from or put to remote. */ +const RemoteLocationTag = z.object({ name: z.string(), radius: z.number(), aSquare: z.number(), bSquare: z.number(), centerPoint: z.object({ - latitude: z.number().nullish().transform(nullToUndefined), - longitude: z.number().nullish().transform(nullToUndefined), + latitude: z.number(), + longitude: z.number(), }), }); +/** Zod schema for the tag that we persist locally. */ +const LocalLocationTag = RemoteLocationTag.extend({ + id: z.string(), +}); + +export type LocationTag = z.infer; + +const saveLocationTags = (tags: LocationTag[]) => + setKV("locationTags", JSON.stringify(tags)); + +/** + * Return all the location tags that are present locally. + * + * Use {@link syncLocationTags} to sync this list with remote. + */ +export const savedLocationTags = async () => + LocalLocationTag.array().parse( + JSON.parse((await getKV("locationTags")) ?? "[]"), + ); + /** * Sync the {@link CGroup} entities that we have locally with remote. * From bcf579e7d7f7d99f7244484e1adee8aeaba377db Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 6 Sep 2024 18:45:41 +0530 Subject: [PATCH 32/42] Don't need the squares --- web/packages/new/photos/services/user-entity.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/web/packages/new/photos/services/user-entity.ts b/web/packages/new/photos/services/user-entity.ts index f433449102..1a040595df 100644 --- a/web/packages/new/photos/services/user-entity.ts +++ b/web/packages/new/photos/services/user-entity.ts @@ -70,8 +70,6 @@ export const syncLocationTags = async () => { const RemoteLocationTag = z.object({ name: z.string(), radius: z.number(), - aSquare: z.number(), - bSquare: z.number(), centerPoint: z.object({ latitude: z.number(), longitude: z.number(), From 1bef528fdec7ac6a1872143a649a54a3bd9d63e0 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 6 Sep 2024 18:49:20 +0530 Subject: [PATCH 33/42] Prune --- web/apps/photos/src/pages/gallery/index.tsx | 1 - .../src/services/locationSearchService.ts | 45 ------------------- web/apps/photos/src/services/searchService.ts | 6 +-- 3 files changed, 3 insertions(+), 49 deletions(-) delete mode 100644 web/apps/photos/src/services/locationSearchService.ts diff --git a/web/apps/photos/src/pages/gallery/index.tsx b/web/apps/photos/src/pages/gallery/index.tsx index 0aa578a5c7..c5e72e5587 100644 --- a/web/apps/photos/src/pages/gallery/index.tsx +++ b/web/apps/photos/src/pages/gallery/index.tsx @@ -99,7 +99,6 @@ import { getSectionSummaries, } from "services/collectionService"; import { syncFiles } from "services/fileService"; -import locationSearchService from "services/locationSearchService"; import { sync, triggerPreFileInfoSync } from "services/sync"; import { syncTrash } from "services/trashService"; import uploadManager from "services/upload/uploadManager"; diff --git a/web/apps/photos/src/services/locationSearchService.ts b/web/apps/photos/src/services/locationSearchService.ts deleted file mode 100644 index 66454965e3..0000000000 --- a/web/apps/photos/src/services/locationSearchService.ts +++ /dev/null @@ -1,45 +0,0 @@ -import log from "@/base/log"; -import type { City } from "@/new/photos/services/search/types"; - -class LocationSearchService { - private cities: Array = []; - private citiesPromise: Promise; - - async loadCities() { - try { - if (this.citiesPromise) { - return; - } - this.citiesPromise = fetch( - "https://static.ente.io/world_cities.json", - ).then((response) => { - return response.json().then((data) => { - this.cities = data["data"]; - }); - }); - await this.citiesPromise; - } catch (e) { - log.error("LocationSearchService loadCities failed", e); - this.citiesPromise = null; - } - } - - async searchCities(searchTerm: string) { - try { - if (!this.citiesPromise) { - this.loadCities(); - } - await this.citiesPromise; - return this.cities.filter((city) => { - return city.name - .toLowerCase() - .startsWith(searchTerm.toLowerCase()); - }); - } catch (e) { - log.error("LocationSearchService searchCities failed", e); - throw e; - } - } -} - -export default new LocationSearchService(); diff --git a/web/apps/photos/src/services/searchService.ts b/web/apps/photos/src/services/searchService.ts index 259dc18e48..08e37ca619 100644 --- a/web/apps/photos/src/services/searchService.ts +++ b/web/apps/photos/src/services/searchService.ts @@ -28,7 +28,6 @@ import { Collection } from "types/collection"; import { EntityType, LocationTag } from "types/entity"; import { getUniqueFiles } from "utils/file"; import { getLatestEntities } from "./entityService"; -import locationSearchService from "./locationSearchService"; // Suggestions shown in the search dropdown's empty state, i.e. when the user // selects the search bar but does not provide any input. @@ -216,8 +215,9 @@ async function getLocationSuggestions(searchPhrase: string) { locationTagSuggestions.map((result) => result.label), ); - const citySearchResults = - await locationSearchService.searchCities(searchPhrase); + const citySearchResults: City[] = []; + // TODO-cgroup + // await locationSearchService.searchCities(searchPhrase); const nonConflictingCityResult = citySearchResults.filter( (city) => !locationTagNames.has(city.name), From eb16c925d2084311e85fab6c47a8615110970be9 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 6 Sep 2024 19:05:49 +0530 Subject: [PATCH 34/42] Prep --- web/apps/photos/src/services/sync.ts | 4 +- .../new/photos/services/search/index.ts | 8 +-- .../new/photos/services/search/types.ts | 6 -- .../new/photos/services/search/worker.ts | 64 +++++++++++++------ .../new/photos/services/user-entity.ts | 2 - 5 files changed, 50 insertions(+), 34 deletions(-) diff --git a/web/apps/photos/src/services/sync.ts b/web/apps/photos/src/services/sync.ts index ce8defd900..0e769fd6d5 100644 --- a/web/apps/photos/src/services/sync.ts +++ b/web/apps/photos/src/services/sync.ts @@ -4,7 +4,7 @@ import { triggerMLStatusSync, triggerMLSync, } from "@/new/photos/services/ml"; -import { triggerSearchDataPrefetchIfNeeded } from "@/new/photos/services/search"; +import { triggerSearchDataSync } from "@/new/photos/services/search"; import { syncLocationTags } from "@/new/photos/services/user-entity"; import { syncEntities } from "services/entityService"; import { syncMapEnabled } from "services/userService"; @@ -42,6 +42,6 @@ export const sync = async () => { syncLocationTags(), syncMapEnabled(), ]); + triggerSearchDataSync(); if (isMLSupported) triggerMLSync(); - triggerSearchDataPrefetchIfNeeded(); }; diff --git a/web/packages/new/photos/services/search/index.ts b/web/packages/new/photos/services/search/index.ts index de3dc0add8..6453bce3b7 100644 --- a/web/packages/new/photos/services/search/index.ts +++ b/web/packages/new/photos/services/search/index.ts @@ -25,13 +25,9 @@ const createComlinkWorker = () => ); /** - * Preload any data that would be needed if the user were to search. - * - * This is an optimization to try and ensure we always have the latest state - * by the time the user gets around to searching. + * Fetch any data that would be needed if the user were to search. */ -export const triggerSearchDataPrefetchIfNeeded = () => - void worker().then((w) => w.prefetchIfNeeded()); +export const triggerSearchDataSync = () => void worker().then((w) => w.sync()); /** * Set the files over which we will search. diff --git a/web/packages/new/photos/services/search/types.ts b/web/packages/new/photos/services/search/types.ts index 5841899ab7..aa77e64aa1 100644 --- a/web/packages/new/photos/services/search/types.ts +++ b/web/packages/new/photos/services/search/types.ts @@ -83,12 +83,6 @@ export type City = Location & { * Name of the city. */ name: string; - /** - * Name of the city, lowercased. - * - * Precomputing this save an lowercasing during the search itself. - */ - lowercasedName: string; }; export enum SuggestionType { diff --git a/web/packages/new/photos/services/search/worker.ts b/web/packages/new/photos/services/search/worker.ts index 031493570e..d7fee7df86 100644 --- a/web/packages/new/photos/services/search/worker.ts +++ b/web/packages/new/photos/services/search/worker.ts @@ -11,6 +11,11 @@ import type { Component } from "chrono-node"; import * as chrono from "chrono-node"; import { expose } from "comlink"; import { z } from "zod"; +import { + savedLocationTags, + syncLocationTags, + type LocationTag, +} from "../user-entity"; import type { City, DateSearchResult, @@ -22,20 +27,47 @@ import type { } from "./types"; import { SuggestionType } from "./types"; +type SearchableCity = City & { + /** + * Name of the city, lowercased. Precomputed to save an op during search. + */ + lowercasedName: string; +}; + +type SearchableLocationTag = LocationTag & { + /** + * Name of the location tag, lowercased. Precomputed to save an op during + * search. + */ + lowercasedName: string; +}; + /** * A web worker that runs the search asynchronously so that the main thread * remains responsive. */ export class SearchWorker { private enteFiles: EnteFile[] = []; - private cities: City[] = []; - private citiesPromise: Promise | undefined; + private locationTags: SearchableLocationTag[] = []; + private cities: SearchableCity[] = []; /** - * Prefetch any data that we might need when the actual search happens. + * Fetch any state we might need when the actual search happens. */ - prefetchIfNeeded() { - this.triggerCityFetchIfNeeded(); + async sync() { + return Promise.allSettled([ + syncLocationTags() + .then(() => savedLocationTags()) + .then((ts) => { + this.locationTags = ts.map((t) => ({ + ...t, + lowercasedName: t.name.toLowerCase(), + })); + }), + fetchCities().then((cs) => { + this.cities = cs; + }), + ]); } /** @@ -53,18 +85,13 @@ export class SearchWorker { locale: string, holidays: DateSearchResult[], ) { - this.prefetchIfNeeded(); - return createSearchQuery(searchString, locale, holidays, this.cities); - } - - /** - * Lazily trigger a fetch of city data, but don't wait for it to complete. - */ - triggerCityFetchIfNeeded() { - if (this.citiesPromise) return; - this.citiesPromise = fetchCities().then((cs) => { - this.cities = cs; - }); + return createSearchQuery( + searchString, + locale, + holidays, + this.locationTags, + this.cities, + ); } /** @@ -81,7 +108,8 @@ const createSearchQuery = async ( searchString: string, locale: string, holidays: DateSearchResult[], - cities: City[], + locationTags: SearchableLocationTag[], + cities: SearchableCity[], ): Promise => { // Normalize it by trimming whitespace and converting to lowercase. const s = searchString.trim().toLowerCase(); diff --git a/web/packages/new/photos/services/user-entity.ts b/web/packages/new/photos/services/user-entity.ts index 1a040595df..9a2af8680d 100644 --- a/web/packages/new/photos/services/user-entity.ts +++ b/web/packages/new/photos/services/user-entity.ts @@ -61,8 +61,6 @@ export const syncLocationTags = async () => { return saveLocationTags([...existingTagsByID.values()]); }; - // TODO-cgroup: Call me - // await removeLegacyDBState(); return syncUserEntity("location", processBatch); }; From 79b2933be757720d46cfa776258c0264d6ce8248 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 6 Sep 2024 19:22:09 +0530 Subject: [PATCH 35/42] Fin 1 --- web/apps/photos/src/services/searchService.ts | 34 ------ .../new/photos/services/search/types.ts | 18 +-- .../new/photos/services/search/worker.ts | 108 ++++++++++-------- 3 files changed, 66 insertions(+), 94 deletions(-) diff --git a/web/apps/photos/src/services/searchService.ts b/web/apps/photos/src/services/searchService.ts index 08e37ca619..8cbadd4efd 100644 --- a/web/apps/photos/src/services/searchService.ts +++ b/web/apps/photos/src/services/searchService.ts @@ -201,40 +201,6 @@ function getFileCaptionSuggestion( }; } -async function getLocationSuggestions(searchPhrase: string) { - const locationTagResults = await searchLocationTag(searchPhrase); - const locationTagSuggestions = locationTagResults.map( - (locationTag) => - ({ - type: SuggestionType.LOCATION, - value: locationTag.data, - label: locationTag.data.name, - }) as Suggestion, - ); - const locationTagNames = new Set( - locationTagSuggestions.map((result) => result.label), - ); - - const citySearchResults: City[] = []; - // TODO-cgroup - // await locationSearchService.searchCities(searchPhrase); - - const nonConflictingCityResult = citySearchResults.filter( - (city) => !locationTagNames.has(city.name), - ); - - const citySearchSuggestions = nonConflictingCityResult.map( - (city) => - ({ - type: SuggestionType.CITY, - value: city, - label: city.name, - }) as Suggestion, - ); - - return [...locationTagSuggestions, ...citySearchSuggestions]; -} - async function getClipSuggestion( searchPhrase: string, ): Promise { diff --git a/web/packages/new/photos/services/search/types.ts b/web/packages/new/photos/services/search/types.ts index aa77e64aa1..b3bf4dad34 100644 --- a/web/packages/new/photos/services/search/types.ts +++ b/web/packages/new/photos/services/search/types.ts @@ -7,6 +7,7 @@ import type { Location } from "@/base/types"; import { FileType } from "@/media/file-type"; import type { MLStatus } from "@/new/photos/services/ml"; import type { EnteFile } from "@/new/photos/types/file"; +import type { LocationTag } from "../user-entity"; export interface DateSearchResult { components: SearchDateComponents; @@ -59,19 +60,6 @@ export interface SearchPerson { // TODO-cgroup: Audit below -export interface LocationOld { - latitude: number | null; - longitude: number | null; -} - -export interface LocationTagData { - name: string; - radius: number; - aSquare: number; - bSquare: number; - centerPoint: LocationOld; -} - /** * A city as identified by a static dataset. * @@ -106,7 +94,7 @@ export interface Suggestion { | number[] | SearchPerson | MLStatus - | LocationTagData + | LocationTag | City | FileType | ClipSearchScores; @@ -115,7 +103,7 @@ export interface Suggestion { export interface SearchQuery { date?: SearchDateComponents; - location?: LocationTagData; + location?: LocationTag; city?: City; collection?: number; files?: number[]; diff --git a/web/packages/new/photos/services/search/worker.ts b/web/packages/new/photos/services/search/worker.ts index d7fee7df86..b6b5f548a9 100644 --- a/web/packages/new/photos/services/search/worker.ts +++ b/web/packages/new/photos/services/search/worker.ts @@ -1,10 +1,7 @@ -// TODO-cgroups -/* eslint-disable @typescript-eslint/no-non-null-assertion */ -/* eslint-disable @typescript-eslint/no-unnecessary-condition */ import { HTTPError } from "@/base/http"; +import type { Location } from "@/base/types"; import { fileCreationPhotoDate, fileLocation } from "@/media/file-metadata"; import type { EnteFile } from "@/new/photos/types/file"; -import { wait } from "@/utils/promise"; import { nullToUndefined } from "@/utils/transform"; import { getPublicMagicMetadataSync } from "@ente/shared/file-metadata"; import type { Component } from "chrono-node"; @@ -19,8 +16,6 @@ import { import type { City, DateSearchResult, - LocationOld, - LocationTagData, SearchDateComponents, SearchQuery, Suggestion, @@ -65,7 +60,10 @@ export class SearchWorker { })); }), fetchCities().then((cs) => { - this.cities = cs; + this.cities = cs.map((c) => ({ + ...c, + lowercasedName: c.name.toLowerCase(), + })); }), ]); } @@ -80,7 +78,7 @@ export class SearchWorker { /** * Convert a search string into a reusable query. */ - async createSearchQuery( + createSearchQuery( searchString: string, locale: string, holidays: DateSearchResult[], @@ -104,20 +102,21 @@ export class SearchWorker { expose(SearchWorker); -const createSearchQuery = async ( +const createSearchQuery = ( searchString: string, locale: string, holidays: DateSearchResult[], locationTags: SearchableLocationTag[], cities: SearchableCity[], -): Promise => { +): Suggestion[] => { // Normalize it by trimming whitespace and converting to lowercase. const s = searchString.trim().toLowerCase(); if (s.length == 0) return []; - // TODO Temp - await wait(0); - return [dateSuggestions(s, locale, holidays)].flat(); + return [ + dateSuggestions(s, locale, holidays), + locationSuggestions(s, locationTags, cities), + ].flat(); }; const dateSuggestions = ( @@ -224,60 +223,82 @@ const fetchCities = async () => { const res = await fetch("https://static.ente.io/world_cities.json"); if (!res.ok) throw new HTTPError(res); return RemoteWorldCities.parse(await res.json()).data.map( - ({ city, lat, lng }) => ({ - name: city, - lowercasedName: city.toLowerCase(), - latitude: lat, - longitude: lng, - }), + ({ city, lat, lng }) => ({ name: city, latitude: lat, longitude: lng }), ); }; -/** - * Return all cities whose name begins with the given search string. - */ -const matchingCities = (s: string, cities: City[]) => - cities.filter(({ lowercasedName }) => lowercasedName.startsWith(s)); +const locationSuggestions = ( + s: string, + locationTags: SearchableLocationTag[], + cities: SearchableCity[], +) => { + const matchingLocationTags = locationTags.filter((t) => + t.lowercasedName.includes(s), + ); + + const matchingLocationTagLNames = new Set( + matchingLocationTags.map((t) => t.lowercasedName), + ); + + const matchingCities = cities.filter( + (c) => + c.lowercasedName.startsWith(s) && + !matchingLocationTagLNames.has(c.lowercasedName), + ); + + return [ + matchingLocationTags.map((t) => ({ + type: SuggestionType.LOCATION, + value: t, + label: t.name, + })), + matchingCities.map((c) => ({ + type: SuggestionType.CITY, + value: c, + label: c.name, + })), + ].flat(); +}; const isMatch = (file: EnteFile, query: SearchQuery) => { - if (query?.collection) { + if (query.collection) { return query.collection === file.collectionID; } - if (query?.date) { + if (query.date) { return isDateComponentsMatch( query.date, fileCreationPhotoDate(file, getPublicMagicMetadataSync(file)), ); } - if (query?.location) { + if (query.location) { const location = fileLocation(file); if (!location) return false; return isInsideLocationTag(location, query.location); } - if (query?.city) { + if (query.city) { const location = fileLocation(file); if (!location) return false; return isInsideCity(location, query.city); } - if (query?.files) { + if (query.files) { return query.files.includes(file.id); } - if (query?.person) { + if (query.person) { return query.person.files.includes(file.id); } - if (typeof query?.fileType !== "undefined") { + if (typeof query.fileType !== "undefined") { return query.fileType === file.metadata.fileType; } - if (typeof query?.clip !== "undefined") { + if (typeof query.clip !== "undefined") { return query.clip.has(file.id); } @@ -305,24 +326,21 @@ const isDateComponentsMatch = ( const defaultCityRadius = 10; const kmsPerDegree = 111.16; -const isInsideLocationTag = ( - location: LocationOld, - locationTag: LocationTagData, -) => isWithinRadius(location, locationTag.centerPoint, locationTag.radius); +const isInsideLocationTag = (location: Location, locationTag: LocationTag) => + isWithinRadius(location, locationTag.centerPoint, locationTag.radius); -const isInsideCity = (location: LocationOld, city: City) => - isWithinRadius(city, location, defaultCityRadius); +const isInsideCity = (location: Location, city: City) => + isWithinRadius(location, city, defaultCityRadius); const isWithinRadius = ( - centerPoint: LocationOld, - location: LocationOld, + location: Location, + center: Location, radius: number, ) => { - const a = - (radius * radiusScaleFactor(centerPoint.latitude!)) / kmsPerDegree; + const a = (radius * radiusScaleFactor(center.latitude)) / kmsPerDegree; const b = radius / kmsPerDegree; - const x = centerPoint.latitude! - location.latitude!; - const y = centerPoint.longitude! - location.longitude!; + const x = center.latitude - location.latitude; + const y = center.longitude - location.longitude; return (x * x) / (a * a) + (y * y) / (b * b) <= 1; }; @@ -331,7 +349,7 @@ const isWithinRadius = ( * search. * * The area bounded by the location tag becomes more elliptical with increase in - * the magnitude of the latitude on the caritesian plane. When latitude is 0 + * the magnitude of the latitude on the cartesian plane. When latitude is 0 * degrees, the ellipse is a circle with a = b = r. When latitude incrases, the * major axis (a) has to be scaled by the secant of the latitude. */ From 90e2dca36b0b525bd15ac27120553fe72a2741a5 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 6 Sep 2024 19:28:43 +0530 Subject: [PATCH 36/42] Fin 2 --- .../Search/SearchBar/searchInput/index.tsx | 4 +- web/apps/photos/src/services/entityService.ts | 193 ------------------ web/apps/photos/src/services/searchService.ts | 22 +- web/apps/photos/src/services/sync.ts | 8 +- web/apps/photos/src/types/entity.ts | 39 ---- web/apps/photos/src/utils/entity.ts | 12 -- web/packages/accounts/pages/verify.tsx | 2 +- .../new/photos/services/search/types.ts | 8 +- .../new/photos/services/search/worker.ts | 5 + 9 files changed, 14 insertions(+), 279 deletions(-) delete mode 100644 web/apps/photos/src/services/entityService.ts delete mode 100644 web/apps/photos/src/types/entity.ts delete mode 100644 web/apps/photos/src/utils/entity.ts diff --git a/web/apps/photos/src/components/Search/SearchBar/searchInput/index.tsx b/web/apps/photos/src/components/Search/SearchBar/searchInput/index.tsx index 0f85391ad0..9edc2c9adf 100644 --- a/web/apps/photos/src/components/Search/SearchBar/searchInput/index.tsx +++ b/web/apps/photos/src/components/Search/SearchBar/searchInput/index.tsx @@ -2,7 +2,6 @@ import { FileType } from "@/media/file-type"; import { isMLEnabled } from "@/new/photos/services/ml"; import type { City, - LocationTagData, SearchDateComponents, SearchPerson, } from "@/new/photos/services/search/types"; @@ -13,6 +12,7 @@ import { SuggestionType, UpdateSearch, } from "@/new/photos/services/search/types"; +import type { LocationTag } from "@/new/photos/services/user-entity"; import { EnteFile } from "@/new/photos/types/file"; import CloseIcon from "@mui/icons-material/Close"; import { IconButton } from "@mui/material"; @@ -126,7 +126,7 @@ export default function SearchInput(props: Iprops) { break; case SuggestionType.LOCATION: search = { - location: selectedOption.value as LocationTagData, + location: selectedOption.value as LocationTag, }; props.setIsOpen(true); break; diff --git a/web/apps/photos/src/services/entityService.ts b/web/apps/photos/src/services/entityService.ts deleted file mode 100644 index 7bd32e556a..0000000000 --- a/web/apps/photos/src/services/entityService.ts +++ /dev/null @@ -1,193 +0,0 @@ -import { decryptMetadataJSON, sharedCryptoWorker } from "@/base/crypto"; -import log from "@/base/log"; -import { apiURL } from "@/base/origins"; -import HTTPService from "@ente/shared/network/HTTPService"; -import localForage from "@ente/shared/storage/localForage"; -import { getToken } from "@ente/shared/storage/localStorage/helpers"; -import { getActualKey } from "@ente/shared/user"; -import { - EncryptedEntity, - EncryptedEntityKey, - Entity, - EntityKey, - EntitySyncDiffResponse, - EntityType, -} from "types/entity"; -import { getLatestVersionEntities } from "utils/entity"; - -const DIFF_LIMIT = 500; - -const ENTITY_TABLES: Record = { - [EntityType.LOCATION_TAG]: "location_tags", -}; - -const ENTITY_KEY_TABLES: Record = { - [EntityType.LOCATION_TAG]: "location_tags_key", -}; - -const ENTITY_SYNC_TIME_TABLES: Record = { - [EntityType.LOCATION_TAG]: "location_tags_time", -}; - -const getLocalEntity = async (type: EntityType) => { - const entities: Array> = - (await localForage.getItem[]>(ENTITY_TABLES[type])) || []; - return entities; -}; - -const getEntityLastSyncTime = async (type: EntityType) => { - return ( - (await localForage.getItem(ENTITY_SYNC_TIME_TABLES[type])) ?? 0 - ); -}; - -const getCachedEntityKey = async (type: EntityType) => { - const entityKey: EntityKey = - (await localForage.getItem(ENTITY_KEY_TABLES[type])) || null; - return entityKey; -}; - -// TODO: unexport -export const getEntityKey = async (type: EntityType) => { - try { - const entityKey = await getCachedEntityKey(type); - if (entityKey) { - return entityKey; - } - const token = getToken(); - if (!token) { - return; - } - const resp = await HTTPService.get( - await apiURL("/user-entity/key"), - { - type, - }, - { - "X-Auth-Token": token, - }, - ); - const encryptedEntityKey: EncryptedEntityKey = resp.data; - const worker = await sharedCryptoWorker(); - const masterKey = await getActualKey(); - const { encryptedKey, header, ...rest } = encryptedEntityKey; - const decryptedData = await worker.decryptB64( - encryptedKey, - header, - masterKey, - ); - const decryptedEntityKey: EntityKey = { data: decryptedData, ...rest }; - localForage.setItem(ENTITY_KEY_TABLES[type], decryptedEntityKey); - return decryptedEntityKey; - } catch (e) { - log.error("Get entity key failed", e); - throw e; - } -}; - -export const getLatestEntities = async (type: EntityType) => { - try { - await syncEntity(type); - return await getLocalEntity(type); - } catch (e) { - log.error("Sync entities failed", e); - throw e; - } -}; - -export const syncEntities = async () => { - try { - await syncEntity(EntityType.LOCATION_TAG); - } catch (e) { - log.error("Sync entities failed", e); - throw e; - } -}; - -const syncEntity = async (type: EntityType): Promise> => { - try { - let entities = await getLocalEntity(type); - log.info( - `Syncing ${type} entities localEntitiesCount: ${entities.length}`, - ); - let syncTime = await getEntityLastSyncTime(type); - log.info(`Syncing ${type} entities syncTime: ${syncTime}`); - let response: EntitySyncDiffResponse; - do { - response = await getEntityDiff(type, syncTime); - if (!response.diff?.length) { - return; - } - - const entityKey = await getEntityKey(type); - // @ts-expect-error TODO: Need to use zod here. - const newDecryptedEntities: Array> = await Promise.all( - response.diff.map(async (entity: EncryptedEntity) => { - if (entity.isDeleted) { - // This entry is deleted, so we don't need to decrypt it, just return it as is - // as unknown as EntityData is a hack to get around the type system - return entity as unknown as Entity; - } - const { encryptedData, header, ...rest } = entity; - const decryptedData = await decryptMetadataJSON({ - encryptedDataB64: encryptedData, - decryptionHeaderB64: header, - keyB64: entityKey.data, - }); - return { - ...rest, - data: decryptedData, - }; - }), - ); - - entities = getLatestVersionEntities([ - ...entities, - ...newDecryptedEntities, - ]); - - const nonDeletedEntities = entities.filter( - (entity) => !entity.isDeleted, - ); - - if (response.diff.length) { - syncTime = response.diff.slice(-1)[0].updatedAt; - } - await localForage.setItem(ENTITY_TABLES[type], nonDeletedEntities); - await localForage.setItem(ENTITY_SYNC_TIME_TABLES[type], syncTime); - log.info( - `Syncing ${type} entities syncedEntitiesCount: ${nonDeletedEntities.length}`, - ); - } while (response.diff.length === DIFF_LIMIT); - } catch (e) { - log.error("Sync entity failed", e); - } -}; - -const getEntityDiff = async ( - type: EntityType, - time: number, -): Promise => { - try { - const token = getToken(); - if (!token) { - return; - } - const resp = await HTTPService.get( - await apiURL("/user-entity/entity/diff"), - { - sinceTime: time, - type, - limit: DIFF_LIMIT, - }, - { - "X-Auth-Token": token, - }, - ); - - return resp.data; - } catch (e) { - log.error("Get entity diff failed", e); - throw e; - } -}; diff --git a/web/apps/photos/src/services/searchService.ts b/web/apps/photos/src/services/searchService.ts index 8cbadd4efd..14965889d3 100644 --- a/web/apps/photos/src/services/searchService.ts +++ b/web/apps/photos/src/services/searchService.ts @@ -16,18 +16,16 @@ import type { import { City, ClipSearchScores, - LocationTagData, SearchOption, SearchQuery, Suggestion, SuggestionType, } from "@/new/photos/services/search/types"; +import type { LocationTag } from "@/new/photos/services/user-entity"; import { EnteFile } from "@/new/photos/types/file"; import { t } from "i18next"; import { Collection } from "types/collection"; -import { EntityType, LocationTag } from "types/entity"; import { getUniqueFiles } from "utils/file"; -import { getLatestEntities } from "./entityService"; // Suggestions shown in the search dropdown's empty state, i.e. when the user // selects the search bar but does not provide any input. @@ -56,7 +54,6 @@ export const getAutoCompleteSuggestions = ...getCollectionSuggestion(searchPhrase, collections), getFileNameSuggestion(searchPhrase, files), getFileCaptionSuggestion(searchPhrase, files), - ...(await getLocationSuggestions(searchPhrase)), ].filter((suggestion) => !!suggestion); return convertSuggestionsToOptions(suggestions); @@ -242,21 +239,6 @@ function searchFilesByCaption(searchPhrase: string, files: EnteFile[]) { ); } -async function searchLocationTag(searchPhrase: string): Promise { - const locationTags = await getLatestEntities( - EntityType.LOCATION_TAG, - ); - const matchedLocationTags = locationTags.filter((locationTag) => - locationTag.data.name.toLowerCase().includes(searchPhrase), - ); - if (matchedLocationTags.length > 0) { - log.info( - `Found ${matchedLocationTags.length} location tags for search phrase`, - ); - } - return matchedLocationTags; -} - const searchClip = async ( searchPhrase: string, ): Promise => { @@ -275,7 +257,7 @@ function convertSuggestionToSearchQuery(option: Suggestion): SearchQuery { case SuggestionType.LOCATION: return { - location: option.value as LocationTagData, + location: option.value as LocationTag, }; case SuggestionType.CITY: diff --git a/web/apps/photos/src/services/sync.ts b/web/apps/photos/src/services/sync.ts index 0e769fd6d5..ac16fa54bf 100644 --- a/web/apps/photos/src/services/sync.ts +++ b/web/apps/photos/src/services/sync.ts @@ -5,8 +5,6 @@ import { triggerMLSync, } from "@/new/photos/services/ml"; import { triggerSearchDataSync } from "@/new/photos/services/search"; -import { syncLocationTags } from "@/new/photos/services/user-entity"; -import { syncEntities } from "services/entityService"; import { syncMapEnabled } from "services/userService"; /** @@ -37,11 +35,7 @@ export const triggerPreFileInfoSync = () => { * before doing the file sync and thus should run immediately after login. */ export const sync = async () => { - await Promise.allSettled([ - syncEntities(), - syncLocationTags(), - syncMapEnabled(), - ]); + await Promise.allSettled([syncMapEnabled()]); triggerSearchDataSync(); if (isMLSupported) triggerMLSync(); }; diff --git a/web/apps/photos/src/types/entity.ts b/web/apps/photos/src/types/entity.ts deleted file mode 100644 index f22a7bac8a..0000000000 --- a/web/apps/photos/src/types/entity.ts +++ /dev/null @@ -1,39 +0,0 @@ -import type { LocationTagData } from "@/new/photos/services/search/types"; - -export enum EntityType { - LOCATION_TAG = "location", -} - -export interface EncryptedEntityKey { - userID: number; - encryptedKey: string; - type: EntityType; - header: string; - createdAt: number; -} - -export interface EntityKey - extends Omit { - data: string; -} - -export interface EncryptedEntity { - id: string; - encryptedData: string; - header: string; - isDeleted: boolean; - createdAt: number; - updatedAt: number; - userID: number; -} - -export type LocationTag = Entity; - -export interface Entity - extends Omit { - data: T; -} - -export interface EntitySyncDiffResponse { - diff: EncryptedEntity[]; -} diff --git a/web/apps/photos/src/utils/entity.ts b/web/apps/photos/src/utils/entity.ts deleted file mode 100644 index 7d8ad28b3c..0000000000 --- a/web/apps/photos/src/utils/entity.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { Entity } from "types/entity"; - -export const getLatestVersionEntities = (entities: Entity[]) => { - const latestVersionEntities = new Map>(); - entities.forEach((entity) => { - const existingEntity = latestVersionEntities.get(entity.id); - if (!existingEntity || existingEntity.updatedAt < entity.updatedAt) { - latestVersionEntities.set(entity.id, entity); - } - }); - return Array.from(latestVersionEntities.values()); -}; diff --git a/web/packages/accounts/pages/verify.tsx b/web/packages/accounts/pages/verify.tsx index 604c36b718..f47d612681 100644 --- a/web/packages/accounts/pages/verify.tsx +++ b/web/packages/accounts/pages/verify.tsx @@ -1,4 +1,5 @@ import type { UserVerificationResponse } from "@/accounts/types/user"; +import log from "@/base/log"; import { ensure } from "@/utils/ensure"; import { VerticallyCentered } from "@ente/shared/components/Container"; import EnteSpinner from "@ente/shared/components/EnteSpinner"; @@ -42,7 +43,6 @@ import { import { configureSRP } from "../services/srp"; import type { PageProps } from "../types/page"; import type { SRPSetupAttributes } from "../types/srp"; -import log from "@/base/log"; const Page: React.FC = ({ appContext }) => { const { logout, showNavBar, setDialogBoxAttributesV2 } = appContext; diff --git a/web/packages/new/photos/services/search/types.ts b/web/packages/new/photos/services/search/types.ts index b3bf4dad34..080803956a 100644 --- a/web/packages/new/photos/services/search/types.ts +++ b/web/packages/new/photos/services/search/types.ts @@ -58,8 +58,6 @@ export interface SearchPerson { displayFaceFile: EnteFile; } -// TODO-cgroup: Audit below - /** * A city as identified by a static dataset. * @@ -67,12 +65,12 @@ export interface SearchPerson { * have information about the city's estimated radius. */ export type City = Location & { - /** - * Name of the city. - */ + /** Name of the city. */ name: string; }; +// TODO-cgroup: Audit below + export enum SuggestionType { DATE = "DATE", LOCATION = "LOCATION", diff --git a/web/packages/new/photos/services/search/worker.ts b/web/packages/new/photos/services/search/worker.ts index b6b5f548a9..6815b7249d 100644 --- a/web/packages/new/photos/services/search/worker.ts +++ b/web/packages/new/photos/services/search/worker.ts @@ -327,6 +327,11 @@ const defaultCityRadius = 10; const kmsPerDegree = 111.16; const isInsideLocationTag = (location: Location, locationTag: LocationTag) => + // This code is included in the photos app which currently doesn't have + // strict mode, and causes a spurious linter warning (but only when included + // in photos!), so we need to ts-ignore. + // eslint-disable-next-line @typescript-eslint/ban-ts-comment, @typescript-eslint/prefer-ts-expect-error + // @ts-ignore isWithinRadius(location, locationTag.centerPoint, locationTag.radius); const isInsideCity = (location: Location, city: City) => From 3d1c1067592b0343ac911cd6486d9c4062c1f355 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 6 Sep 2024 19:50:58 +0530 Subject: [PATCH 37/42] allSettled swallows errors all still runs to completion, it just rejects early --- web/apps/photos/src/services/sync.ts | 2 +- web/packages/base/blob-cache.ts | 5 ++--- web/packages/new/photos/services/migrations.ts | 4 ++-- web/packages/new/photos/services/search/worker.ts | 2 +- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/web/apps/photos/src/services/sync.ts b/web/apps/photos/src/services/sync.ts index ac16fa54bf..af11377762 100644 --- a/web/apps/photos/src/services/sync.ts +++ b/web/apps/photos/src/services/sync.ts @@ -35,7 +35,7 @@ export const triggerPreFileInfoSync = () => { * before doing the file sync and thus should run immediately after login. */ export const sync = async () => { - await Promise.allSettled([syncMapEnabled()]); + await Promise.all([syncMapEnabled()]); triggerSearchDataSync(); if (isMLSupported) triggerMLSync(); }; diff --git a/web/packages/base/blob-cache.ts b/web/packages/base/blob-cache.ts index dae10f7725..49a1fb91e7 100644 --- a/web/packages/base/blob-cache.ts +++ b/web/packages/base/blob-cache.ts @@ -237,9 +237,8 @@ export const clearBlobCaches = async () => { return isElectron() ? clearOPFSCaches() : clearWebCaches(); }; -const clearWebCaches = async () => { - await Promise.allSettled(blobCacheNames.map((name) => caches.delete(name))); -}; +const clearWebCaches = () => + Promise.all(blobCacheNames.map((name) => caches.delete(name))); const clearOPFSCaches = async () => { const root = await navigator.storage.getDirectory(); diff --git a/web/packages/new/photos/services/migrations.ts b/web/packages/new/photos/services/migrations.ts index c4b0df2183..0c4e18d9cf 100644 --- a/web/packages/new/photos/services/migrations.ts +++ b/web/packages/new/photos/services/migrations.ts @@ -43,7 +43,7 @@ export const runMigrations = async () => { // Added: Aug 2024 (v1.7.3). Prunable. const m0 = () => - Promise.allSettled([ + Promise.all([ // Delete the legacy face DB v1. deleteDB("mldata"), @@ -71,7 +71,7 @@ const m1 = () => // Older versions of the user-entities code kept the diff related state // in a different place. These entries are not needed anymore (the tags // themselves will get resynced). - Promise.allSettled([ + Promise.all([ localForage.removeItem("location_tags"), localForage.removeItem("location_tags_key"), localForage.removeItem("location_tags_time"), diff --git a/web/packages/new/photos/services/search/worker.ts b/web/packages/new/photos/services/search/worker.ts index 6815b7249d..8d29f2bb70 100644 --- a/web/packages/new/photos/services/search/worker.ts +++ b/web/packages/new/photos/services/search/worker.ts @@ -50,7 +50,7 @@ export class SearchWorker { * Fetch any state we might need when the actual search happens. */ async sync() { - return Promise.allSettled([ + return Promise.all([ syncLocationTags() .then(() => savedLocationTags()) .then((ts) => { From c66202381903acc5cd23ccdfc5e306f1e214f1d2 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 6 Sep 2024 20:55:26 +0530 Subject: [PATCH 38/42] Mirror the parse --- web/apps/photos/src/services/export/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/apps/photos/src/services/export/index.ts b/web/apps/photos/src/services/export/index.ts index c65b7ceadc..84d48832a9 100644 --- a/web/apps/photos/src/services/export/index.ts +++ b/web/apps/photos/src/services/export/index.ts @@ -1413,9 +1413,9 @@ const getGoogleLikeMetadataFile = ( dateTimeFormatter: Intl.DateTimeFormat, ) => { const metadata: Metadata = file.metadata; - const creationTime = Math.floor(metadata.creationTime / 1000000); + const creationTime = Math.floor(metadata.creationTime / 1e6); const modificationTime = Math.floor( - (metadata.modificationTime ?? metadata.creationTime) / 1000000, + (metadata.modificationTime ?? metadata.creationTime) / 1e6, ); const result: Record = { title: fileExportName, From dd53cf4e58d996d706dda5a7a1557a858547d048 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 7 Sep 2024 06:05:42 +0530 Subject: [PATCH 39/42] Handle NaNs --- .../photos/src/services/upload/takeout.ts | 5 +++-- web/packages/utils/parse.ts | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 web/packages/utils/parse.ts diff --git a/web/apps/photos/src/services/upload/takeout.ts b/web/apps/photos/src/services/upload/takeout.ts index de0f475b65..02bff3cd25 100644 --- a/web/apps/photos/src/services/upload/takeout.ts +++ b/web/apps/photos/src/services/upload/takeout.ts @@ -6,6 +6,7 @@ import log from "@/base/log"; import { type Location } from "@/base/types"; import type { UploadItem } from "@/new/photos/services/upload/types"; import { readStream } from "@/new/photos/utils/native-stream"; +import { maybeParseInt } from "@/utils/parse"; /** * The data we read from the JSON metadata sidecar files. @@ -144,8 +145,8 @@ const parseGTTimestamp = (o: unknown): number | undefined => { "timestamp" in o && typeof o.timestamp == "string" ) { - const { timestamp } = o; - if (timestamp) return parseInt(timestamp, 10) * 1e6; + const timestamp = maybeParseInt(o.timestamp); + if (timestamp) return timestamp * 1e6; } return undefined; }; diff --git a/web/packages/utils/parse.ts b/web/packages/utils/parse.ts new file mode 100644 index 0000000000..5e0f3c0739 --- /dev/null +++ b/web/packages/utils/parse.ts @@ -0,0 +1,19 @@ +/** + * A wrapper over parseInt that deals with its sheNaNigans. + * + * This function takes as an input a string nominally (though the implementation + * is meant to work for arbitrary JavaScript values). It parses it into a base + * 10 integer. If the result is NaN, it returns undefined, otherwise it returns + * the parsed integer. + * + * From MDN: + * + * > To be sure that you are working with numbers, coerce the value to a number + * > and use Number.isNaN() to test the result() + * > + * > https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/isNaN + */ +export const maybeParseInt = (s: string) => { + const n = parseInt(s, 10); + return Number.isNaN(n) ? undefined : n; +}; From b70444acac71811cc2ccd7d5433a2b089384f594 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 7 Sep 2024 10:07:55 +0530 Subject: [PATCH 40/42] Update docs --- web/packages/base/crypto/index.ts | 3 ++- web/packages/base/crypto/libsodium.ts | 2 +- web/packages/base/session-store.ts | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/web/packages/base/crypto/index.ts b/web/packages/base/crypto/index.ts index eceea2408d..48f7a9526b 100644 --- a/web/packages/base/crypto/index.ts +++ b/web/packages/base/crypto/index.ts @@ -193,7 +193,8 @@ export const encryptMetadataJSON = async (r: { : sharedCryptoWorker().then((w) => w.encryptMetadataJSON(r)); /** - * Decrypt a box encrypted using {@link encryptBoxB64}. + * Decrypt a box encrypted using {@link encryptBoxB64} and returns the decrypted + * bytes. */ export const decryptBox = (box: EncryptedBox, key: BytesOrB64) => inWorker() diff --git a/web/packages/base/crypto/libsodium.ts b/web/packages/base/crypto/libsodium.ts index 25ff0057e9..225b5efe72 100644 --- a/web/packages/base/crypto/libsodium.ts +++ b/web/packages/base/crypto/libsodium.ts @@ -390,7 +390,7 @@ export async function encryptFileChunk( } /** - * Decrypt the result of {@link encryptBoxB64}. + * Decrypt the result of {@link encryptBoxB64} and return the decrypted bytes. */ export const decryptBox = async ( { encryptedData, nonce }: EncryptedBox, diff --git a/web/packages/base/session-store.ts b/web/packages/base/session-store.ts index 2493467348..989375941d 100644 --- a/web/packages/base/session-store.ts +++ b/web/packages/base/session-store.ts @@ -3,7 +3,7 @@ import { decryptBox } from "./crypto"; import { toB64 } from "./crypto/libsodium"; /** - * Return the user's master key (as a base64 string) from session storage. + * Return the user's master key from session storage. * * Precondition: The user should be logged in. */ From 6159f5e4eef73da3b874ff78c811307a6a71c64a Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 7 Sep 2024 10:11:44 +0530 Subject: [PATCH 41/42] cen --- .../new/photos/services/user-entity.ts | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/web/packages/new/photos/services/user-entity.ts b/web/packages/new/photos/services/user-entity.ts index 9a2af8680d..c134f038a3 100644 --- a/web/packages/new/photos/services/user-entity.ts +++ b/web/packages/new/photos/services/user-entity.ts @@ -327,16 +327,19 @@ const userEntityDiff = async ( * See also, [Note: User entity keys]. */ const getOrCreateEntityKeyB64 = async (type: EntityType) => { + // Get the user's master key (we need it to encrypt/decrypt the entity key). + const masterKey = await masterKeyFromSession(); + // See if we already have it locally. const saved = await savedRemoteUserEntityKey(type); - if (saved) return decryptEntityKey(saved); + if (saved) return decryptEntityKey(saved, masterKey); // See if remote already has it. const existing = await getUserEntityKey(type); if (existing) { // Only save it if we can decrypt it to avoid corrupting our local state // in unforeseen circumstances. - const result = await decryptEntityKey(existing); + const result = await decryptEntityKey(existing, masterKey); await saveRemoteUserEntityKey(type, existing); return result; } @@ -346,8 +349,8 @@ const getOrCreateEntityKeyB64 = async (type: EntityType) => { // As a sanity check, genarate the key but immediately encrypt it as if it // were fetched from remote and then try to decrypt it before doing anything // with it. - const generated = await generateNewEncryptedEntityKey(); - const result = decryptEntityKey(generated); + const generated = await generateNewEncryptedEntityKey(masterKey); + const result = decryptEntityKey(generated, masterKey); await postUserEntityKey(type, generated); await saveRemoteUserEntityKey(type, generated); return result; @@ -378,10 +381,10 @@ const saveRemoteUserEntityKey = ( * Generate a new entity key and return it in the shape of an * {@link RemoteUserEntityKey} after encrypting it using the user's master key. */ -const generateNewEncryptedEntityKey = async () => { +const generateNewEncryptedEntityKey = async (masterKey: Uint8Array) => { const { encryptedData, nonce } = await encryptBoxB64( await generateNewBlobOrStreamKey(), - await masterKeyFromSession(), + masterKey, ); // Remote calls it the header, but it really is the nonce. return { encryptedKey: encryptedData, header: nonce }; @@ -390,14 +393,17 @@ const generateNewEncryptedEntityKey = async () => { /** * Decrypt an encrypted entity key using the user's master key. */ -const decryptEntityKey = async (remote: RemoteUserEntityKey) => +const decryptEntityKey = async ( + remote: RemoteUserEntityKey, + masterKey: Uint8Array, +) => decryptBoxB64( { encryptedData: remote.encryptedKey, // Remote calls it the header, but it really is the nonce. nonce: remote.header, }, - await masterKeyFromSession(), + masterKey, ); /** From 762bf413e823d58b79a3035dc876f567bd85fe5c Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 7 Sep 2024 10:19:20 +0530 Subject: [PATCH 42/42] Web workers cannot access session store --- .../new/photos/services/search/index.ts | 4 ++- .../new/photos/services/search/worker.ts | 7 +++-- .../new/photos/services/user-entity.ts | 29 ++++++++++++------- 3 files changed, 27 insertions(+), 13 deletions(-) diff --git a/web/packages/new/photos/services/search/index.ts b/web/packages/new/photos/services/search/index.ts index 6453bce3b7..497b817a40 100644 --- a/web/packages/new/photos/services/search/index.ts +++ b/web/packages/new/photos/services/search/index.ts @@ -1,3 +1,4 @@ +import { masterKeyFromSession } from "@/base/session-store"; import { ComlinkWorker } from "@/base/worker/comlink-worker"; import i18n, { t } from "i18next"; import type { EnteFile } from "../../types/file"; @@ -27,7 +28,8 @@ const createComlinkWorker = () => /** * Fetch any data that would be needed if the user were to search. */ -export const triggerSearchDataSync = () => void worker().then((w) => w.sync()); +export const triggerSearchDataSync = () => + void worker().then((w) => masterKeyFromSession().then((k) => w.sync(k))); /** * Set the files over which we will search. diff --git a/web/packages/new/photos/services/search/worker.ts b/web/packages/new/photos/services/search/worker.ts index 8d29f2bb70..423b20382b 100644 --- a/web/packages/new/photos/services/search/worker.ts +++ b/web/packages/new/photos/services/search/worker.ts @@ -48,10 +48,13 @@ export class SearchWorker { /** * Fetch any state we might need when the actual search happens. + * + * @param masterKey The user's master key. Web workers do not have access to + * session storage so this key needs to be passed to us explicitly. */ - async sync() { + async sync(masterKey: Uint8Array) { return Promise.all([ - syncLocationTags() + syncLocationTags(masterKey) .then(() => savedLocationTags()) .then((ts) => { this.locationTags = ts.map((t) => ({ diff --git a/web/packages/new/photos/services/user-entity.ts b/web/packages/new/photos/services/user-entity.ts index c134f038a3..7064d21d82 100644 --- a/web/packages/new/photos/services/user-entity.ts +++ b/web/packages/new/photos/services/user-entity.ts @@ -7,7 +7,6 @@ import { import { authenticatedRequestHeaders, ensureOk, HTTPError } from "@/base/http"; import { getKV, getKVN, setKV } from "@/base/kv"; import { apiURL } from "@/base/origins"; -import { masterKeyFromSession } from "@/base/session-store"; import { ensure } from "@/utils/ensure"; import { nullToUndefined } from "@/utils/transform"; import { z } from "zod"; @@ -41,8 +40,11 @@ export type EntityType = * This function fetches all the location tag user entities from remote and * updates our local database. It uses local state to remember the last time it * synced, so each subsequent sync is a lightweight diff. + * + * @param masterKey The user's master key. This is used to encrypt and decrypt + * the location tags specific entity key. */ -export const syncLocationTags = async () => { +export const syncLocationTags = async (masterKey: Uint8Array) => { const decoder = new TextDecoder(); const parse = (id: string, data: Uint8Array): LocationTag => ({ id, @@ -61,7 +63,7 @@ export const syncLocationTags = async () => { return saveLocationTags([...existingTagsByID.values()]); }; - return syncUserEntity("location", processBatch); + return syncUserEntity("location", masterKey, processBatch); }; /** Zod schema for the tag that we get from or put to remote. */ @@ -102,8 +104,11 @@ export const savedLocationTags = async () => * checked. * * This diff is then applied to the data we have persisted locally. + * + * @param masterKey The user's master key. This is used to encrypt and decrypt + * the cgroup specific entity key. */ -export const syncCGroups = () => { +export const syncCGroups = (masterKey: Uint8Array) => { const parse = async (id: string, data: Uint8Array): Promise => { const rp = RemoteCGroup.parse(JSON.parse(await gunzip(data))); return { @@ -125,7 +130,7 @@ export const syncCGroups = () => { ), ); - return syncUserEntity("cgroup", processBatch); + return syncUserEntity("cgroup", masterKey, processBatch); }; const RemoteCGroup = z.object({ @@ -200,12 +205,16 @@ interface UserEntityChange { * * For each diff response, the {@link processBatch} is invoked to give a chance * to caller to apply the updates to the data we have persisted locally. + * + * The user's {@link masterKey} is used to decrypt (or encrypt, when generating + * a new one) the entity key. */ const syncUserEntity = async ( type: EntityType, + masterKey: Uint8Array, processBatch: (entities: UserEntityChange[]) => Promise, ) => { - const entityKeyB64 = await getOrCreateEntityKeyB64(type); + const entityKeyB64 = await getOrCreateEntityKeyB64(type, masterKey); let sinceTime = (await savedLatestUpdatedAt(type)) ?? 0; // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition, no-constant-condition @@ -326,10 +335,10 @@ const userEntityDiff = async ( * * See also, [Note: User entity keys]. */ -const getOrCreateEntityKeyB64 = async (type: EntityType) => { - // Get the user's master key (we need it to encrypt/decrypt the entity key). - const masterKey = await masterKeyFromSession(); - +const getOrCreateEntityKeyB64 = async ( + type: EntityType, + masterKey: Uint8Array, +) => { // See if we already have it locally. const saved = await savedRemoteUserEntityKey(type); if (saved) return decryptEntityKey(saved, masterKey);