diff --git a/web/packages/new/photos/services/ml/index.ts b/web/packages/new/photos/services/ml/index.ts
index 6797a52b77..a2045b8447 100644
--- a/web/packages/new/photos/services/ml/index.ts
+++ b/web/packages/new/photos/services/ml/index.ts
@@ -41,15 +41,18 @@ let _isMLEnabled = false;
 let _comlinkWorker: ComlinkWorker<typeof MLWorker> | undefined;
 
 /** Lazily created, cached, instance of {@link MLWorker}. */
-export const worker = async () =>
-    (_comlinkWorker ??= createComlinkWorker()).remote;
+export const worker = async () => {
+    if (!_comlinkWorker) _comlinkWorker = await createComlinkWorker();
+    return _comlinkWorker.remote;
+};
 
-const createComlinkWorker = () => {
+const createComlinkWorker = async () => {
     const cw = new ComlinkWorker<typeof MLWorker>(
         "ml",
         new Worker(new URL("worker.ts", import.meta.url)),
     );
-    void cw.remote.then((w) => getUserAgent().then((ua) => w.init(ua)));
+    const ua = await getUserAgent();
+    await cw.remote.then((w) => w.init(ua));
     return cw;
 };
 
@@ -196,6 +199,7 @@ export const setIsFaceIndexingEnabled = (enabled: boolean) =>
  *
  * @param count Limit the resulting list of indexable files to {@link count}.
  */
+// TODO-ML: Move to worker
 export const syncWithLocalFilesAndGetFilesToIndex = async (
     userID: number,
     count: number,
diff --git a/web/packages/new/photos/services/ml/worker.ts b/web/packages/new/photos/services/ml/worker.ts
index be99ca6055..5434407c98 100644
--- a/web/packages/new/photos/services/ml/worker.ts
+++ b/web/packages/new/photos/services/ml/worker.ts
@@ -3,7 +3,9 @@ import type { FaceIndex } from "@/new/photos/services/ml/types";
 import type { EnteFile } from "@/new/photos/types/file";
 import log from "@/next/log";
 // import { expose } from "comlink";
+import downloadManager from "@/new/photos/services/download";
 import { getKVN } from "@/next/kv";
+import { ensureAuthToken } from "@/next/local-user";
 import { ensure } from "@/utils/ensure";
 import { wait } from "@/utils/promise";
 import { syncWithLocalFilesAndGetFilesToIndex } from ".";
@@ -52,8 +54,11 @@ export class MLWorker {
      * @param userAgent The user agent string to use as the client field in the
      * embeddings generated during indexing by this client.
      */
-    init(userAgent: string) {
+    async init(userAgent: string) {
         this.userAgent = userAgent;
+        // Initialize the downloadManager running in the web worker with the
+        // user's token. It'll be used to download files to index if needed.
+        await downloadManager.init(await ensureAuthToken());
     }
 
     /**