From d37675cc5768357b155e504d2d08656d099eb73f Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 30 Jan 2025 08:19:48 +0530 Subject: [PATCH] [desktop] im => vips More cleanup is pending, once we give this nightly a go. --- desktop/.gitignore | 6 ++- desktop/docs/dependencies.md | 2 + desktop/package.json | 2 +- desktop/scripts/beforeBuild.js | 20 ++++---- desktop/scripts/magick.js | 78 ------------------------------ desktop/scripts/vips.js | 68 ++++++++++++++++++++++++++ desktop/src/main/services/image.ts | 38 ++++++++++++--- 7 files changed, 117 insertions(+), 97 deletions(-) delete mode 100755 desktop/scripts/magick.js create mode 100755 desktop/scripts/vips.js diff --git a/desktop/.gitignore b/desktop/.gitignore index f8ca01608c..23cd8677d7 100644 --- a/desktop/.gitignore +++ b/desktop/.gitignore @@ -21,5 +21,9 @@ out # electron-builder dist/ -# We download it on demand, if needed for the particular OS/arch. +# Legacy, people who checked out main for a brief while in Jan 2025 may have +# this. Can be removed in the future. build/magick* + +# We download it on demand, if needed for the particular OS/arch. +build/vips* diff --git a/desktop/docs/dependencies.md b/desktop/docs/dependencies.md index 1a01be69f3..62a9a3eae0 100644 --- a/desktop/docs/dependencies.md +++ b/desktop/docs/dependencies.md @@ -111,6 +111,8 @@ For video conversions and metadata extraction, we use ffmpeg. To bundle a > ffmpeg binary and using the wasm one (that our renderer process already has). > Which is why we bundle it to speed up operations on the desktop app. +TODO: Cleanup im - update this + On Linux and Windows, we use ImageMagick for thumbnail generation and JPEG conversion of unpreviewable images. A static OS/architecture specific binary of this is bundled in our extra resources (`build`) folder by `scripts/magick.sh` diff --git a/desktop/package.json b/desktop/package.json index 42317d2478..21cdc94551 100644 --- a/desktop/package.json +++ b/desktop/package.json @@ -19,7 +19,7 @@ "postinstall": "electron-builder install-app-deps", "lint": "yarn prettier --check --log-level warn . && yarn eslint && yarn tsc", "lint-fix": "yarn prettier --write --log-level warn . && yarn eslint && yarn tsc", - "prepare": "node scripts/magick.js" + "prepare": "node scripts/vips.js" }, "resolutions": { "jackspeak": "2.1.1" diff --git a/desktop/scripts/beforeBuild.js b/desktop/scripts/beforeBuild.js index 052544004d..ba4d3ada30 100755 --- a/desktop/scripts/beforeBuild.js +++ b/desktop/scripts/beforeBuild.js @@ -3,8 +3,8 @@ const fsp = require("fs/promises"); /** * This hook is invoked during the initial build (e.g. when triggered by "yarn * build"), and importantly, on each rebuild for a different architecture during - * the build. We use it to ensure that the magick binary is for the current - * architecture being built. See "[Note: ImageMagick]" for more details. + * the build. We use it to ensure that the vips binary is for the current + * architecture being built. See "[Note: vips]" for more details. * * The documentation for this hook is at: * https://www.electron.build/app-builder-lib.interface.configuration#beforebuild @@ -22,8 +22,10 @@ const fsp = require("fs/promises"); * }, * arch: 'arm64' * - * Note that we must not return falsey from this function, because - * > Resolving to false will skip dependencies install or rebuild. + * Note that we must not return falsey from this function, because: + * + * > Resolving to false will skip dependencies install or rebuild. + * */ module.exports = async (context) => { const { appDir, platform, arch } = context; @@ -42,7 +44,7 @@ module.exports = async (context) => { const download = async (downloadName, outputName) => { const out = `${appDir}/build/${outputName}`; console.log(`Downloading ${downloadName}`); - const downloadPath = `https://github.com/ente-io/ImageMagick/releases/download/2025-01-21/${downloadName}`; + const downloadPath = `https://github.com/ente-io/libvips-packaging/releases/tag/v8.16.0/${downloadName}`; return fetch(downloadPath) .then((res) => res.blob()) .then((blob) => fsp.writeFile(out, blob.stream())) @@ -51,13 +53,13 @@ module.exports = async (context) => { switch (`${platform.nodeName}-${arch}`) { case "linux-x64": - await download("magick-x86_64", "magick"); + return download("vips-x64", "vips"); case "linux-arm64": - await download("magick-aarch64", "magick"); + return download("vips-arm64", "vips"); case "win32-x64": - await download("magick-x64.exe", "magick.exe"); + return download("vips-x86_64.exe", "vips.exe"); case "linux-arm64": - await download("magick-arm64.exe", "magick.exe"); + return download("vips-aarch64.exe", "vips.exe"); } return true; diff --git a/desktop/scripts/magick.js b/desktop/scripts/magick.js deleted file mode 100755 index 8cd457e580..0000000000 --- a/desktop/scripts/magick.js +++ /dev/null @@ -1,78 +0,0 @@ -/** - * [Note: ImageMagick] - * - * We need static builds for Linux and Windows for both x64 and ARM. For this, - * we need a custom workflow because (as of writing): - * - * 1. Upstream doesn't publish ARM64 binaries for Linux - * - * 2. The Windows portable releases are not part of the artifacts attached to - * the upstream GitHub release. - * - * Our custom workflow is an adaption of the upstream release.yml - its goal is - * to have 4 standalone binaries - Linux x64, Linux ARM, Win x64, Win ARM - - * attached to a GitHub release from which we can pull them when building the - * desktop app. - * - * This is our custom workflow, which runs on a fork of upstream: - * https://github.com/ente-io/ImageMagick/commit/df895cce13d6a3f874a716c05ff2babeb33351b9 - * (For reference, we also include a copy of it in this repo - `magick.yml`). - * - * The binaries it creates are available at - * https://github.com/ente-io/ImageMagick/releases/tag/2025-01-21. - * - * To integrate this ImageMagick binary, we need to modify two places: - * - * 1. This script, `magick.js`, runs during "yarn install" (it is set as the - * "prepare" step in our `package.json`). It downloads the relevant binary - * for the current OS/arch combination and places it in the `build` folder, - * allowing it to be used during development. - * - * 2. The sibling script, `beforeBuild.js`, runs during "yarn build" (it is set - * as the beforeBuild script in `electrons-builder.yml`). It downloads the - * relevant binary for the OS/arch combination being built. - * - * Note that `magick.js` would've already run once `beforeBuild.js` is run, but - * on our CI we prepare builds for multiple architectures in one go, so we need - * to unconditonally replace the binary with the relevant one for the current - * architecture being built (which might be different from the one we're running - * on). `beforeBuild.js` runs for each architecture being built. - * - * On macOS, we don't need ImageMagick since there we use the native `sips`. - */ - -const fs = require("fs"); -const fsp = require("fs/promises"); - -const main = () => { - switch (`${process.platform}-${process.arch}`) { - case "linux-x64": - return downloadIfNeeded("magick-x86_64", "magick"); - case "linux-arm64": - return downloadIfNeeded("magick-aarch64", "magick"); - case "win32-x64": - return downloadIfNeeded("magick-x64.exe", "magick.exe"); - case "linux-arm64": - return downloadIfNeeded("magick-arm64.exe", "magick.exe"); - } -}; - -const downloadIfNeeded = (downloadName, outputName) => { - const out = `build/${outputName}`; - - try { - // Making the file executable is the last step, so if the file exists at - // this path and is executable, we assume it is the correct one. - fs.accessSync(out, fs.constants.X_OK); - return; - } catch {} - - console.log(`Downloading ${downloadName}`); - const downloadPath = `https://github.com/ente-io/ImageMagick/releases/download/2025-01-21/${downloadName}`; - return fetch(downloadPath) - .then((res) => res.blob()) - .then((blob) => fsp.writeFile(out, blob.stream())) - .then(() => fsp.chmod(out, "744")); -}; - -main(); diff --git a/desktop/scripts/vips.js b/desktop/scripts/vips.js new file mode 100755 index 0000000000..0e29ca1a46 --- /dev/null +++ b/desktop/scripts/vips.js @@ -0,0 +1,68 @@ +/** + * [Note: vips] + * + * For use within our Electron app we need static builds for Linux and Windows + * for both x64 and ARM. For this, we need a custom workflow because (as of + * writing) upstream doesn't publish these. + * + * This is our custom workflow, which runs on a fork of upstream: + * https://github.com/ente-io/libvips-packaging/commit/a298aff3e1f25f713508d31d0c3a55a4f828fdd3 + * + * The binaries it creates are available at + * https://github.com/ente-io/libvips-packaging/releases/tag/v8.16.0 + * + * To integrate this binary, we need to modify two places: + * + * 1. This script, `vips.js`, runs during "yarn install" (it is set as the + * "prepare" step in our `package.json`). It downloads the relevant binary + * for the current OS/arch combination and places it in the `build` folder, + * allowing it to be used during development. + * + * 2. The sibling script, `beforeBuild.js`, runs during "yarn build" (it is set + * as the beforeBuild script in `electrons-builder.yml`). It downloads the + * relevant binary for the OS/arch combination being built. + * + * Note that `vips.js` would've already run once `beforeBuild.js` is run, but on + * our CI we prepare builds for multiple architectures in one go, so we need to + * unconditonally replace the binary with the relevant one for the current + * architecture being built (which might be different from the one we're running + * on). `beforeBuild.js` runs for each architecture being built. + * + * On macOS, we don't need `vips` since there we use the native `sips`. + */ + +const fs = require("fs"); +const fsp = require("fs/promises"); + +const main = () => { + switch (`${process.platform}-${process.arch}`) { + case "linux-x64": + return downloadIfNeeded("vips-x64", "vips"); + case "linux-arm64": + return downloadIfNeeded("vips-arm64", "vips"); + case "win32-x64": + return downloadIfNeeded("vips-x86_64.exe", "vips.exe"); + case "linux-arm64": + return downloadIfNeeded("vips-aarch64.exe", "vips.exe"); + } +}; + +const downloadIfNeeded = (downloadName, outputName) => { + const out = `build/${outputName}`; + + try { + // Making the file executable is the last step, so if the file exists at + // this path and is executable, we assume it is the correct one. + fs.accessSync(out, fs.constants.X_OK); + return; + } catch {} + + console.log(`Downloading ${downloadName}`); + const downloadPath = `https://github.com/ente-io/libvips-packaging/releases/tag/v8.16.0/${downloadName}`; + return fetch(downloadPath) + .then((res) => res.blob()) + .then((blob) => fsp.writeFile(out, blob.stream())) + .then(() => fsp.chmod(out, "744")); +}; + +main(); diff --git a/desktop/src/main/services/image.ts b/desktop/src/main/services/image.ts index aa0d99b457..83bf437635 100644 --- a/desktop/src/main/services/image.ts +++ b/desktop/src/main/services/image.ts @@ -45,14 +45,17 @@ const convertToJPEGCommand = ( case "linux": case "win32": - return [ - imageMagickPath(), - "convert", - inputFilePath, - "-quality", - "100%", - outputFilePath, - ]; + return [vipsPath(), "copy", inputFilePath, outputFilePath]; + + // TODO: Cleanup im + // return [ + // imageMagickPath(), + // "convert", + // inputFilePath, + // "-quality", + // "100%", + // outputFilePath, + // ]; default: throw new Error("Not available on the current OS/arch"); @@ -68,6 +71,15 @@ const imageMagickPath = () => process.platform == "win32" ? "magick.exe" : "magick", ); +/** + * Path to the vips executable bundled with our app on Linux and Windows. + */ +const vipsPath = () => + path.join( + isDev ? "build" : process.resourcesPath, + process.platform == "win32" ? "vips.exe" : "vips", + ); + export const generateImageThumbnail = async ( dataOrPathOrZipItem: Uint8Array | string | ZipItem, maxDimension: number, @@ -138,6 +150,16 @@ const generateImageThumbnailCommand = ( case "linux": case "win32": + return [ + vipsPath(), + "thumbnail", + inputFilePath, + `${outputFilePath}[Q=${quality}]`, + `${maxDimension}`, + ]; + + case "aix" /* dummy case to hold TODO */: + /* TODO: Cleanup im */ return [ imageMagickPath(), "convert",