[desktop] Fix OOM on large library imports (#3847)
It is hard for me to be certain, but I feel this should resolve the sporadic OOMs that have been reported when uploading large libraries. - https://github.com/ente-io/ente/issues/2500 - https://github.com/ente-io/ente/discussions/3420 There are two fixes here: 1. First one is a inefficient array concat in our code. This was not incorrect per se, but it did lead to an allocation pattern that caused V8's GC to crash the renderer with OOMs. 2. But even after the first fix, I was able to sometimes reproduce OOMs. I added a lot of instrumentation (I've cherry-committed some of it to git history for future reference when debugging similar issues), but I couldn't spot any abnormal allocation patterns during uploads. Out of ideas, I started imagining it was a Chromium issue, and on a whim, I updated Electron 30 => 33 (something I needed to do anyway, as part of regular app dependency updates). That apparently has resolved the remaining OOMs. With these changes, I've not been able to reproduce a crash even after bumping up the parallel upload count from 4 to 12. I've let the parallel upload count be at the existing 4 for now, but if indeed we stop getting field reports of OOM crashes after this is released, we can increase that too in the future.
This commit is contained in:
@@ -48,7 +48,7 @@
|
||||
"ajv": "^8.17.1",
|
||||
"concurrently": "^8.2.2",
|
||||
"cross-env": "^7.0.3",
|
||||
"electron": "^30.4.0",
|
||||
"electron": "^33.0.2",
|
||||
"electron-builder": "^25.0.5",
|
||||
"eslint": "^9",
|
||||
"prettier": "^3.3.3",
|
||||
|
||||
@@ -1282,10 +1282,10 @@ electron-updater@^6.3.4:
|
||||
semver "^7.6.3"
|
||||
tiny-typed-emitter "^2.1.0"
|
||||
|
||||
electron@^30.4.0:
|
||||
version "30.4.0"
|
||||
resolved "https://registry.yarnpkg.com/electron/-/electron-30.4.0.tgz#66641a644059147f0e597e49999599e23dcdbfe3"
|
||||
integrity sha512-ric3KLPQ9anXYjtTDkj5NbEcXZqRUwqxrxTviIjLdMdHqd5O+hkSHEzXgbSJUOt+7uw+zZuybn9+IM9y7iEpqg==
|
||||
electron@^33.0.2:
|
||||
version "33.0.2"
|
||||
resolved "https://registry.yarnpkg.com/electron/-/electron-33.0.2.tgz#db31b105bf0edd7c8600dfb70c2dfc214e3789f1"
|
||||
integrity sha512-C2WksfP0COsMHbYXSJG68j6S3TjuGDrw/YT42B526yXalIlNQZ2GeAYKryg6AEMkIp3p8TUfDRD0+HyiyCt/nw==
|
||||
dependencies:
|
||||
"@electron/get" "^2.0.0"
|
||||
"@types/node" "^20.9.0"
|
||||
|
||||
@@ -37,6 +37,7 @@ import {
|
||||
} from "@/new/photos/services/upload/types";
|
||||
import { detectFileTypeInfoFromChunk } from "@/new/photos/utils/detect-type";
|
||||
import { readStream } from "@/new/photos/utils/native-stream";
|
||||
import { mergeUint8Arrays } from "@/utils/array";
|
||||
import { ensure, ensureInteger, ensureNumber } from "@/utils/ensure";
|
||||
import { CustomError, handleUploadError } from "@ente/shared/error";
|
||||
import { addToCollection } from "services/collectionService";
|
||||
@@ -1573,9 +1574,7 @@ async function combineChunksToFormUploadPart(
|
||||
if (done) {
|
||||
break;
|
||||
}
|
||||
for (let index = 0; index < chunk.length; index++) {
|
||||
combinedChunks.push(chunk[index]);
|
||||
}
|
||||
combinedChunks.push(chunk);
|
||||
}
|
||||
return Uint8Array.from(combinedChunks);
|
||||
return mergeUint8Arrays(combinedChunks);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user