From b1c62f6c933cb228b93ea1cd07732dcc5bfbd9b7 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 22 Jan 2025 07:37:21 +0530 Subject: [PATCH] [desktop] IM 7 - Handle CI rebuilds, and add windows extension - CI builds both arch binaries in one go, so the singular yarn install hook is not enough - Without the extension windows doesn't run it --- desktop/.gitignore | 2 +- desktop/docs/dependencies.md | 3 +- desktop/electron-builder.yml | 1 + desktop/eslint.config.mjs | 8 +--- desktop/scripts/beforeBuild.js | 60 ++++++++++++++++++++++++++++++ desktop/scripts/magick.js | 58 +++++++++++++++++------------ desktop/src/main/services/image.ts | 5 ++- 7 files changed, 103 insertions(+), 34 deletions(-) create mode 100755 desktop/scripts/beforeBuild.js diff --git a/desktop/.gitignore b/desktop/.gitignore index cf2a1d92f6..f8ca01608c 100644 --- a/desktop/.gitignore +++ b/desktop/.gitignore @@ -22,4 +22,4 @@ out dist/ # We download it on demand, if needed for the particular OS/arch. -build/magick +build/magick* diff --git a/desktop/docs/dependencies.md b/desktop/docs/dependencies.md index fc05d70ccb..1a01be69f3 100644 --- a/desktop/docs/dependencies.md +++ b/desktop/docs/dependencies.md @@ -113,7 +113,8 @@ For video conversions and metadata extraction, we use ffmpeg. To bundle a On Linux and Windows, we use ImageMagick for thumbnail generation and JPEG conversion of unpreviewable images. A static OS/architecture specific binary of -this is bundled in our extra resources (`build`) folder by `scripts/magick.sh`. +this is bundled in our extra resources (`build`) folder by `scripts/magick.sh` +and/or `scripts/beforeBuild.js`. See "[Note: ImageMagick]" for more details. On macOS, we use the `sips` CLI tool for these tasks, but that is already available on the host machine, and is not bundled with our app. diff --git a/desktop/electron-builder.yml b/desktop/electron-builder.yml index 6e3df57656..463ac4fc5b 100644 --- a/desktop/electron-builder.yml +++ b/desktop/electron-builder.yml @@ -6,6 +6,7 @@ files: extraFiles: - from: build to: resources +beforeBuild: scripts/beforeBuild.js protocols: - name: Ente schemes: ["ente"] diff --git a/desktop/eslint.config.mjs b/desktop/eslint.config.mjs index 4175c52a95..b2b2ca3b9a 100644 --- a/desktop/eslint.config.mjs +++ b/desktop/eslint.config.mjs @@ -19,13 +19,7 @@ export default ts.config( { // The list of (minimatch) globs to ignore. This needs to be the only // key in this configuration object. - ignores: [ - "eslint.config.mjs", - "scripts/magick.js", - "app/", - "out/", - "dist/", - ], + ignores: ["eslint.config.mjs", "scripts/*.js", "app/", "out/", "dist/"], }, { // Rule customizations. diff --git a/desktop/scripts/beforeBuild.js b/desktop/scripts/beforeBuild.js new file mode 100755 index 0000000000..91b7247f15 --- /dev/null +++ b/desktop/scripts/beforeBuild.js @@ -0,0 +1,60 @@ +const fsp = require("fs/promises"); + +/** + * This hook is invoked during the initial build (e.g. when triggered by "yarn + * build"), and importantly, on each rebuild for a different architecture during + * the build. We use it to ensure that the magick binary is for the current + * architecture being built. See "[Note: ImageMagick]" for more details. + * + * The documentation for this hook is at: + * https://www.electron.build/app-builder-lib.interface.configuration#beforebuild + * + * > The function to be run before dependencies are installed or rebuilt. + * + * Here is an example of the context that it gets + * https://www.electron.build/app-builder-lib.interface.beforebuildcontext + * + * appDir: '/path/to/ente/desktop', + * platform: Platform { + * name: 'mac', + * buildConfigurationKey: 'mac', + * nodeName: 'darwin' + * }, + * arch: 'arm64' + * + */ +module.exports = async (context) => { + const { appDir, platform, arch } = context; + + // The arch used by Electron Builder is not the same as the arch used by + // Node's process, but for the two cases that we care about, "x64" and + // "arm64", both of them use the string constant and thus can be compared. + // + // https://github.com/electron-userland/electron-builder/blob/master/packages/builder-util/src/arch.ts#L9 + // https://nodejs.org/api/process.html#processarch + if (arch == process.arch) { + // `magick.js` would've already downloaded the file, nothing to do. + return; + } + + const download = async (downloadName, outputName) => { + const out = `${appDir}/build/${outputName}`; + console.log(`Downloading ${downloadName}`); + const downloadPath = `https://github.com/ente-io/ImageMagick/releases/download/2025-01-21/${downloadName}`; + return fetch(downloadPath) + .then((res) => res.blob()) + .then((blob) => fsp.writeFile(out, blob.stream())) + .then(() => fsp.chmod(out, "744")); + }; + + switch (`${platform.nodeName}-${arch}`) { + case "linux-x64": + return download("magick-x86_64", "magick"); + case "linux-arm64": + return download("magick-aarch64", "magick"); + case "win32-x64": + return download("magick-x64.exe", "magick.exe"); + case "linux-arm64": + return download("magick-arm64.exe", "magick.exe"); + } +}; diff --git a/desktop/scripts/magick.js b/desktop/scripts/magick.js index 80b8214750..8cd457e580 100755 --- a/desktop/scripts/magick.js +++ b/desktop/scripts/magick.js @@ -1,5 +1,5 @@ /** - * ## ImageMagick + * [Note: ImageMagick] * * We need static builds for Linux and Windows for both x64 and ARM. For this, * we need a custom workflow because (as of writing): @@ -21,18 +21,45 @@ * The binaries it creates are available at * https://github.com/ente-io/ImageMagick/releases/tag/2025-01-21. * - * This script downloads the relevant binary for the current OS/arch combination - * and places it in the `build` folder. This script runs whenever "yarn install" - * is called as it is set as the "prepare" step in our `package.json`. + * To integrate this ImageMagick binary, we need to modify two places: * - * On macOS, we don't need ImageMagick since Apple ships `sips`. + * 1. This script, `magick.js`, runs during "yarn install" (it is set as the + * "prepare" step in our `package.json`). It downloads the relevant binary + * for the current OS/arch combination and places it in the `build` folder, + * allowing it to be used during development. + * + * 2. The sibling script, `beforeBuild.js`, runs during "yarn build" (it is set + * as the beforeBuild script in `electrons-builder.yml`). It downloads the + * relevant binary for the OS/arch combination being built. + * + * Note that `magick.js` would've already run once `beforeBuild.js` is run, but + * on our CI we prepare builds for multiple architectures in one go, so we need + * to unconditonally replace the binary with the relevant one for the current + * architecture being built (which might be different from the one we're running + * on). `beforeBuild.js` runs for each architecture being built. + * + * On macOS, we don't need ImageMagick since there we use the native `sips`. */ const fs = require("fs"); const fsp = require("fs/promises"); const main = () => { - const out = "build/magick"; + switch (`${process.platform}-${process.arch}`) { + case "linux-x64": + return downloadIfNeeded("magick-x86_64", "magick"); + case "linux-arm64": + return downloadIfNeeded("magick-aarch64", "magick"); + case "win32-x64": + return downloadIfNeeded("magick-x64.exe", "magick.exe"); + case "linux-arm64": + return downloadIfNeeded("magick-arm64.exe", "magick.exe"); + } +}; + +const downloadIfNeeded = (downloadName, outputName) => { + const out = `build/${outputName}`; + try { // Making the file executable is the last step, so if the file exists at // this path and is executable, we assume it is the correct one. @@ -40,26 +67,9 @@ const main = () => { return; } catch {} - let downloadName = (() => { - switch (`${process.platform}-${process.arch}`) { - case "linux-x64": - return "magick-x86_64"; - case "linux-arm64": - return "magick-aarch64"; - case "win32-x64": - return "magick-x64.exe"; - case "linux-arm64": - return "magick-arm64.exe"; - default: - return undefined; - } - })(); - - if (!downloadName) return; - console.log(`Downloading ${downloadName}`); const downloadPath = `https://github.com/ente-io/ImageMagick/releases/download/2025-01-21/${downloadName}`; - void fetch(downloadPath) + return fetch(downloadPath) .then((res) => res.blob()) .then((blob) => fsp.writeFile(out, blob.stream())) .then(() => fsp.chmod(out, "744")); diff --git a/desktop/src/main/services/image.ts b/desktop/src/main/services/image.ts index 1dc37d9726..aa0d99b457 100644 --- a/desktop/src/main/services/image.ts +++ b/desktop/src/main/services/image.ts @@ -63,7 +63,10 @@ const convertToJPEGCommand = ( * Path to the magick executable bundled with our app on Linux and Windows. */ const imageMagickPath = () => - path.join(isDev ? "build" : process.resourcesPath, "magick"); + path.join( + isDev ? "build" : process.resourcesPath, + process.platform == "win32" ? "magick.exe" : "magick", + ); export const generateImageThumbnail = async ( dataOrPathOrZipItem: Uint8Array | string | ZipItem,