From 2690b874ee5763bbadd97363edf9d026e5c71620 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 1 May 2024 19:49:09 +0530 Subject: [PATCH 01/25] Make the migration a no-op --- web/apps/photos/src/utils/storage/mlIDbStorage.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/web/apps/photos/src/utils/storage/mlIDbStorage.ts b/web/apps/photos/src/utils/storage/mlIDbStorage.ts index 40e6dad662..766c3ac9a9 100644 --- a/web/apps/photos/src/utils/storage/mlIDbStorage.ts +++ b/web/apps/photos/src/utils/storage/mlIDbStorage.ts @@ -144,7 +144,13 @@ class MLIDbStorage { .objectStore("configs") .add(DEFAULT_ML_SEARCH_CONFIG, ML_SEARCH_CONFIG_NAME); } + /* + This'll go in version 5. Note that version 4 was never released, + but it was in main for a while, so we'll just skip it to avoid + breaking the upgrade path for people who ran the mainline. + */ if (oldVersion < 4) { + /* try { await tx .objectStore("configs") @@ -163,8 +169,8 @@ class MLIDbStorage { // the shipped implementation should have a more // deterministic migration. } + */ } - log.info( `ML DB upgraded from version ${oldVersion} to version ${newVersion}`, ); From 0226a99fa3fdebb52d8a449bfc4175de167b7a6b Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 1 May 2024 20:09:36 +0530 Subject: [PATCH 02/25] Disable enabling ML search --- .../src/components/ml/MLSearchSettings.tsx | 46 ++++++++++++------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/web/apps/photos/src/components/ml/MLSearchSettings.tsx b/web/apps/photos/src/components/ml/MLSearchSettings.tsx index 583b79529c..9b50c2d6ae 100644 --- a/web/apps/photos/src/components/ml/MLSearchSettings.tsx +++ b/web/apps/photos/src/components/ml/MLSearchSettings.tsx @@ -22,7 +22,7 @@ import { getFaceSearchEnabledStatus, updateFaceSearchEnabledStatus, } from "services/userService"; -import { openLink } from "utils/common"; +import { isInternalUser } from "utils/user"; export const MLSearchSettings = ({ open, onClose, onRootClose }) => { const { @@ -255,8 +255,8 @@ function EnableFaceSearch({ open, onClose, enableFaceSearch, onRootClose }) { } function EnableMLSearch({ onClose, enableMlSearch, onRootClose }) { - const showDetails = () => - openLink("https://ente.io/blog/desktop-ml-beta", true); + // const showDetails = () => + // openLink("https://ente.io/blog/desktop-ml-beta", true); return ( @@ -269,25 +269,37 @@ function EnableMLSearch({ onClose, enableMlSearch, onRootClose }) { {" "} - + {/* */} +

+ We're putting finishing touches, coming back soon! +

+

+ + Existing indexed faces will continue to show. + +

- - - + {/* + - + > + {t("ML_MORE_DETAILS")} + + */} + + )} ); From 5ba2e35af6e71d5f4e4823fc5e03655f48494e46 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 1 May 2024 20:13:08 +0530 Subject: [PATCH 03/25] Force disable it for non internal users --- .../photos/src/utils/machineLearning/config.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/web/apps/photos/src/utils/machineLearning/config.ts b/web/apps/photos/src/utils/machineLearning/config.ts index 4d2030ca3e..30a65b8f1d 100644 --- a/web/apps/photos/src/utils/machineLearning/config.ts +++ b/web/apps/photos/src/utils/machineLearning/config.ts @@ -10,6 +10,7 @@ import mlIDbStorage, { ML_SYNC_CONFIG_NAME, ML_SYNC_JOB_CONFIG_NAME, } from "utils/storage/mlIDbStorage"; +import { isInternalUser } from "utils/user"; export async function getMLSyncJobConfig() { return mlIDbStorage.getConfig( @@ -23,10 +24,15 @@ export async function getMLSyncConfig() { } export async function getMLSearchConfig() { - return mlIDbStorage.getConfig( - ML_SEARCH_CONFIG_NAME, - DEFAULT_ML_SEARCH_CONFIG, - ); + if (isInternalUser()) { + return mlIDbStorage.getConfig( + ML_SEARCH_CONFIG_NAME, + DEFAULT_ML_SEARCH_CONFIG, + ); + } + // Force disabled for everyone else while we finalize it to avoid redundant + // reindexing for users. + return DEFAULT_ML_SEARCH_CONFIG; } export async function updateMLSyncJobConfig(newConfig: JobConfig) { From 30f22e333abc8cf84578aa1aa12dc2b515db3eff Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 1 May 2024 21:04:41 +0530 Subject: [PATCH 04/25] Pass file when we have it --- .../src/services/upload/uploadManager.ts | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/web/apps/photos/src/services/upload/uploadManager.ts b/web/apps/photos/src/services/upload/uploadManager.ts index 3d53adbeaf..38fd7037be 100644 --- a/web/apps/photos/src/services/upload/uploadManager.ts +++ b/web/apps/photos/src/services/upload/uploadManager.ts @@ -609,11 +609,25 @@ class UploadManager { ].includes(uploadResult) ) { try { + let file: File | undefined; + const uploadItem = + uploadableItem.uploadItem ?? + uploadableItem.livePhotoAssets.image; + if (uploadItem) { + if (uploadItem instanceof File) { + file = uploadItem; + } else if ( + typeof uploadItem == "string" || + Array.isArray(uploadItem) + ) { + // path from desktop, no file object + } else { + file = uploadItem.file; + } + } eventBus.emit(Events.FILE_UPLOADED, { enteFile: decryptedFile, - localFile: - uploadableItem.uploadItem ?? - uploadableItem.livePhotoAssets.image, + localFile: file, }); } catch (e) { log.warn("Ignoring error in fileUploaded handlers", e); From cd5c1e35fa32c5c3474de66b512246a1eafb9764 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 1 May 2024 21:05:29 +0530 Subject: [PATCH 05/25] Disable live clip, rely on the thumbnailed version --- web/apps/photos/src/pages/gallery/index.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/apps/photos/src/pages/gallery/index.tsx b/web/apps/photos/src/pages/gallery/index.tsx index 70b48c3cc6..20d95ce00f 100644 --- a/web/apps/photos/src/pages/gallery/index.tsx +++ b/web/apps/photos/src/pages/gallery/index.tsx @@ -370,7 +370,7 @@ export default function Gallery() { syncWithRemote(false, true); }, SYNC_INTERVAL_IN_MICROSECONDS); if (electron) { - void clipService.setupOnFileUploadListener(); + // void clipService.setupOnFileUploadListener(); electron.onMainWindowFocus(() => syncWithRemote(false, true)); } }; From 8327c2b8816ff6763d03e03cc4235c929f38f38c Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 2 May 2024 10:23:31 +0530 Subject: [PATCH 06/25] Remove unused ElectronFile --- .../photos/src/components/Upload/Uploader.tsx | 3 +- web/apps/photos/src/services/upload/types.ts | 12 ++++++- web/packages/next/types/file.ts | 36 ------------------- 3 files changed, 12 insertions(+), 39 deletions(-) delete mode 100644 web/packages/next/types/file.ts diff --git a/web/apps/photos/src/components/Upload/Uploader.tsx b/web/apps/photos/src/components/Upload/Uploader.tsx index 53ad1fb32f..7174306556 100644 --- a/web/apps/photos/src/components/Upload/Uploader.tsx +++ b/web/apps/photos/src/components/Upload/Uploader.tsx @@ -1,6 +1,5 @@ import { basename } from "@/next/file"; import log from "@/next/log"; -import { type FileAndPath } from "@/next/types/file"; import type { CollectionMapping, Electron, ZipItem } from "@/next/types/ipc"; import { CustomError } from "@ente/shared/error"; import { isPromise } from "@ente/shared/utils"; @@ -20,7 +19,7 @@ import { getPublicCollectionUploaderName, savePublicCollectionUploaderName, } from "services/publicCollectionService"; -import type { UploadItem } from "services/upload/types"; +import type { FileAndPath, UploadItem } from "services/upload/types"; import type { InProgressUpload, SegregatedFinishedUploads, diff --git a/web/apps/photos/src/services/upload/types.ts b/web/apps/photos/src/services/upload/types.ts index 05ad332d4a..25e2ab408a 100644 --- a/web/apps/photos/src/services/upload/types.ts +++ b/web/apps/photos/src/services/upload/types.ts @@ -1,4 +1,3 @@ -import type { FileAndPath } from "@/next/types/file"; import type { ZipItem } from "@/next/types/ipc"; /** @@ -30,6 +29,17 @@ import type { ZipItem } from "@/next/types/ipc"; */ export type UploadItem = File | FileAndPath | string | ZipItem; +/** + * When we are running in the context of our desktop app, we have access to the + * absolute path of {@link File} objects. This convenience type clubs these two + * bits of information, saving us the need to query the path again and again + * using the {@link getPathForFile} method of {@link Electron}. + */ +export interface FileAndPath { + file: File; + path: string; +} + /** * The of cases of {@link UploadItem} that apply when we're running in the * context of our desktop app. diff --git a/web/packages/next/types/file.ts b/web/packages/next/types/file.ts deleted file mode 100644 index 6dd1032cdb..0000000000 --- a/web/packages/next/types/file.ts +++ /dev/null @@ -1,36 +0,0 @@ -/* - * ElectronFile is a custom interface that is used to represent - * any file on disk as a File-like object in the Electron desktop app. - * - * This was added to support the auto-resuming of failed uploads - * which needed absolute paths to the files which the - * normal File interface does not provide. - */ -export interface ElectronFile { - name: string; - path: string; - size: number; - lastModified: number; - stream: () => Promise>; - blob: () => Promise; - arrayBuffer: () => Promise; -} - -/** - * When we are running in the context of our desktop app, we have access to the - * absolute path of {@link File} objects. This convenience type clubs these two - * bits of information, saving us the need to query the path again and again - * using the {@link getPathForFile} method of {@link Electron}. - */ -export interface FileAndPath { - file: File; - path: string; -} - -export interface EventQueueItem { - type: "upload" | "trash"; - folderPath: string; - collectionName?: string; - paths?: string[]; - files?: ElectronFile[]; -} From 68721b8168b7292d32a7319d21f67e620aa8e3aa Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 2 May 2024 11:10:26 +0530 Subject: [PATCH 07/25] Pick from the correct table --- web/apps/photos/src/services/embeddingService.ts | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/web/apps/photos/src/services/embeddingService.ts b/web/apps/photos/src/services/embeddingService.ts index a4309e314c..36af848424 100644 --- a/web/apps/photos/src/services/embeddingService.ts +++ b/web/apps/photos/src/services/embeddingService.ts @@ -86,7 +86,11 @@ export const syncEmbeddings = async () => { allLocalFiles.forEach((file) => { fileIdToKeyMap.set(file.id, file.key); }); - await cleanupDeletedEmbeddings(allLocalFiles, allEmbeddings); + await cleanupDeletedEmbeddings( + allLocalFiles, + allEmbeddings, + EMBEDDINGS_TABLE, + ); log.info(`Syncing embeddings localCount: ${allEmbeddings.length}`); for (const model of models) { let modelLastSinceTime = await getModelEmbeddingSyncTime(model); @@ -168,7 +172,11 @@ export const syncFileEmbeddings = async () => { allLocalFiles.forEach((file) => { fileIdToKeyMap.set(file.id, file.key); }); - await cleanupDeletedEmbeddings(allLocalFiles, allEmbeddings); + await cleanupDeletedEmbeddings( + allLocalFiles, + allEmbeddings, + FILE_EMBEDING_TABLE, + ); log.info(`Syncing embeddings localCount: ${allEmbeddings.length}`); for (const model of models) { let modelLastSinceTime = await getModelEmbeddingSyncTime(model); @@ -289,6 +297,7 @@ export const putEmbedding = async ( export const cleanupDeletedEmbeddings = async ( allLocalFiles: EnteFile[], allLocalEmbeddings: Embedding[] | FileML[], + tableName: string, ) => { const activeFileIds = new Set(); allLocalFiles.forEach((file) => { @@ -302,6 +311,6 @@ export const cleanupDeletedEmbeddings = async ( log.info( `cleanupDeletedEmbeddings embeddingsCount: ${allLocalEmbeddings.length} remainingEmbeddingsCount: ${remainingEmbeddings.length}`, ); - await localForage.setItem(EMBEDDINGS_TABLE, remainingEmbeddings); + await localForage.setItem(tableName, remainingEmbeddings); } }; From fa182b951dbb81e8fd0b8485d4b8aa4aecf16a74 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 2 May 2024 12:52:05 +0530 Subject: [PATCH 08/25] [desktop] Resurrect build Untested --- .github/workflows/desktop-release.yml | 72 +++++++++++++++++++++++++ desktop/.github/workflows/build.yml | 55 ------------------- desktop/README.md | 6 --- desktop/docs/release.md | 78 ++++++++++----------------- 4 files changed, 99 insertions(+), 112 deletions(-) create mode 100644 .github/workflows/desktop-release.yml delete mode 100644 desktop/.github/workflows/build.yml diff --git a/.github/workflows/desktop-release.yml b/.github/workflows/desktop-release.yml new file mode 100644 index 0000000000..44c63e5b23 --- /dev/null +++ b/.github/workflows/desktop-release.yml @@ -0,0 +1,72 @@ +name: "Release (photos desktop)" + +# This will create a new draft release with public artifacts. +# +# Note that a release will only get created if there is an associated tag +# (GitHub releases need a corresponding tag). + +on: + workflow_dispatch: # Allow manually running the action + push: + # Run when a tag matching the pattern "photosd-v*"" is pushed + # See: [Note: Testing release workflows that are triggered by tags] + tags: + - "photosd-v*" + +jobs: + release: + runs-on: ${{ matrix.os }} + + defaults: + run: + working-directory: desktop + + strategy: + matrix: + os: [macos-latest, ubuntu-latest, windows-latest] + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Setup node + uses: actions/setup-node@v4 + with: + node-version: 20 + + - name: Install dependencies + run: yarn install + + - name: Prepare for app notarization + if: startsWith(matrix.os, 'macos') + # Import Apple API key for app notarization on macOS + run: | + mkdir -p ~/private_keys/ + echo '${{ secrets.APPLE_API_KEY }}' > ~/private_keys/AuthKey_${{ secrets.APPLE_API_KEY_ID }}.p8 + + - name: Install libarchive-tools for pacman build + if: startsWith(matrix.os, 'ubuntu') + # See: + # https://github.com/electron-userland/electron-builder/issues/4181 + run: sudo apt-get install libarchive-tools + + - name: Build + uses: ente-io/action-electron-builder@v1.0.0 + with: + # GitHub token, automatically provided to the action + # (No need to define this secret in the repo settings) + github_token: ${{ secrets.GITHUB_TOKEN }} + + # If the commit is tagged with a version (e.g. "v1.0.0"), + # release the app after building + release: ${{ startsWith(github.ref, 'refs/tags/v') }} + + mac_certs: ${{ secrets.MAC_CERTS }} + mac_certs_password: ${{ secrets.MAC_CERTS_PASSWORD }} + env: + # macOS notarization API key details + API_KEY_ID: ${{ secrets.APPLE_API_KEY_ID }} + API_KEY_ISSUER_ID: ${{ secrets.APPLE_API_KEY_ISSUER_ID }} + USE_HARD_LINKS: false diff --git a/desktop/.github/workflows/build.yml b/desktop/.github/workflows/build.yml deleted file mode 100644 index acd744c056..0000000000 --- a/desktop/.github/workflows/build.yml +++ /dev/null @@ -1,55 +0,0 @@ -name: Build/release - -on: - push: - tags: - - v* - -jobs: - release: - runs-on: ${{ matrix.os }} - - strategy: - matrix: - os: [macos-latest, ubuntu-latest, windows-latest] - - steps: - - name: Check out Git repository - uses: actions/checkout@v3 - with: - submodules: recursive - - - name: Install Node.js, NPM and Yarn - uses: actions/setup-node@v3 - with: - node-version: 20 - - - name: Prepare for app notarization - if: startsWith(matrix.os, 'macos') - # Import Apple API key for app notarization on macOS - run: | - mkdir -p ~/private_keys/ - echo '${{ secrets.api_key }}' > ~/private_keys/AuthKey_${{ secrets.api_key_id }}.p8 - - - name: Install libarchive-tools for pacman build # Related https://github.com/electron-userland/electron-builder/issues/4181 - if: startsWith(matrix.os, 'ubuntu') - run: sudo apt-get install libarchive-tools - - - name: Ente Electron Builder Action - uses: ente-io/action-electron-builder@v1.0.0 - with: - # GitHub token, automatically provided to the action - # (No need to define this secret in the repo settings) - github_token: ${{ secrets.github_token }} - - # If the commit is tagged with a version (e.g. "v1.0.0"), - # release the app after building - release: ${{ startsWith(github.ref, 'refs/tags/v') }} - - mac_certs: ${{ secrets.mac_certs }} - mac_certs_password: ${{ secrets.mac_certs_password }} - env: - # macOS notarization API key - API_KEY_ID: ${{ secrets.api_key_id }} - API_KEY_ISSUER_ID: ${{ secrets.api_key_issuer_id}} - USE_HARD_LINKS: false diff --git a/desktop/README.md b/desktop/README.md index 05149f5d0c..39b7663fab 100644 --- a/desktop/README.md +++ b/desktop/README.md @@ -10,12 +10,6 @@ To know more about Ente, see [our main README](../README.md) or visit ## Building from source -> [!CAUTION] -> -> We're improving the security of the desktop app further by migrating to -> Electron's sandboxing and contextIsolation. These updates are still WIP and -> meanwhile the instructions below might not fully work on the main branch. - Fetch submodules ```sh diff --git a/desktop/docs/release.md b/desktop/docs/release.md index 7254e26fc1..0a5c2970f5 100644 --- a/desktop/docs/release.md +++ b/desktop/docs/release.md @@ -1,43 +1,33 @@ ## Releases -> [!NOTE] -> -> TODO(MR): This document needs to be audited and changed as we do the first -> release from this new monorepo. - The Github Action that builds the desktop binaries is triggered by pushing a tag -matching the pattern `photos-desktop-v1.2.3`. This value should match the -version in `package.json`. +matching the pattern `photosd-v1.2.3`. This value should match the version in +`package.json`. -So the process for doing a release would be. +To make a new release -1. Create a new branch (can be named anything). On this branch, include your - changes. +1. Create a new branch (can be named anything). On this branch, change the + `version` in `package.json` to `1.x.x` and finalize `CHANGELOG.md`. -2. Mention the changes in `CHANGELOG.md`. - -3. Changing the `version` in `package.json` to `1.x.x`. - -4. Commit and push to remote +2. Commit, tag and push to remote. Note that the tag should have a `photosd-` + prefix: ```sh - git add package.json && git commit -m 'Release v1.x.x' - git tag v1.x.x - git push && git push --tags + git add CHANGELOG.md package.json + git commit -m 'Release v1.x.x' + git tag photosd-v1.x.x + git push origin photosd-v1.x.x ``` -This by itself will already trigger a new release. The GitHub action will create -a new draft release that can then be used as descibed below. + This will trigger the GitHub action that will create a new draft release. -To wrap up, we also need to merge back these changes into main. So for that, +3. To wrap up, increase the version number in `package.json` the next release + train. That is, suppose we just released `v4.0.1`. Then we'll change the + version number in main to `v4.0.2-beta.0`. Each pre-release will modify the + `beta.0` part. Finally, at the time of the next release, this'll become + `v4.0.2`. -5. Open a PR for the branch that we're working on (where the above tag was - pushed from) to get it merged into main. - -6. In this PR, also increase the version number for the next release train. That - is, supposed we just released `v4.0.1`. Then we'll change the version number - in main to `v4.0.2-next.0`. Each pre-release will modify the `next.0` part. - Finally, at the time of the next release, this'll become `v4.0.2`. +4. Open a PR for the branch to get it merged into main. The GitHub Action runs on Windows, Linux and macOS. It produces the artifacts defined in the `build` value in `package.json`. @@ -49,26 +39,14 @@ defined in the `build` value in `package.json`. Additionally, the GitHub action notarizes the macOS DMG. For this it needs credentials provided via GitHub secrets. -During the build the Sentry webpack plugin checks to see if SENTRY_AUTH_TOKEN is -defined. If so, it uploads the sourcemaps for the renderer process to Sentry -(For our GitHub action, the SENTRY_AUTH_TOKEN is defined as a GitHub secret). +To rollout the build, we need to publish the draft release. This needs to be +done in the old photos-desktop repository since that the Electron Updater +mechanism doesn't work well with monorepos. So we need to create a new tag with +changelog updates on +[photos-desktop](https://github.com/ente-io/photos-desktop/), use that to create +a new release, copying over all the artifacts. -The sourcemaps for the main (node) process are currently not sent to Sentry -(this works fine in practice since the node process files are not minified, we -only run `tsc`). - -Once the build is done, a draft release with all these artifacts attached is -created. The build is idempotent, so if something goes wrong and we need to -re-run the GitHub action, just delete the draft release (if it got created) and -start a new run by pushing a new tag (if some code changes are required). - -If no code changes are required, say the build failed for some transient network -or sentry issue, we can even be re-run by the build by going to Github Action -age and rerun from there. This will re-trigger for the same tag. - -If everything goes well, we'll have a release on GitHub, and the corresponding -source maps for the renderer process uploaded to Sentry. There isn't anything -else to do: +Thereafter, everything is automated: - The website automatically redirects to the latest release on GitHub when people try to download. @@ -76,7 +54,7 @@ else to do: - The file formats with support auto update (Windows `exe`, the Linux AppImage and the macOS DMG) also check the latest GitHub release automatically to download and apply the update (the rest of the formats don't support auto - updates). + updates yet). - We're not putting the desktop app in other stores currently. It is available as a `brew cask`, but we only had to open a PR to add the initial formula, @@ -87,6 +65,4 @@ else to do: We can also publish the draft releases by checking the "pre-release" option. Such releases don't cause any of the channels (our website, or the desktop app auto updater, or brew) to be notified, instead these are useful for giving links -to pre-release builds to customers. Generally, in the version number for these -we'll add a label to the version, e.g. the "beta.x" in `1.x.x-beta.x`. This -should be done both in `package.json`, and what we tag the commit with. +to pre-release builds to customers. From aed781b0ffe8b54cc857f232c4b01ebe31190c11 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 2 May 2024 19:05:52 +0530 Subject: [PATCH 09/25] Use same credentials as the auth app While we won't actually be using the monorepo for releases, get the action to a known state: - MAC_OS_CERTIFICATE and MAC_OS_CERTIFICATE_PASSWORD is the same GitHub secret that the auth app already uses - Need to add APPLE_API_KEY, APPLE_API_KEY_ID, APPLE_API_KEY_ISSUER_ID. --- .github/workflows/desktop-release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/desktop-release.yml b/.github/workflows/desktop-release.yml index 44c63e5b23..60f012b653 100644 --- a/.github/workflows/desktop-release.yml +++ b/.github/workflows/desktop-release.yml @@ -63,8 +63,8 @@ jobs: # release the app after building release: ${{ startsWith(github.ref, 'refs/tags/v') }} - mac_certs: ${{ secrets.MAC_CERTS }} - mac_certs_password: ${{ secrets.MAC_CERTS_PASSWORD }} + mac_certs: ${{ secrets.MAC_OS_CERTIFICATE }} + mac_certs_password: ${{ secrets.MAC_OS_CERTIFICATE_PASSWORD }} env: # macOS notarization API key details API_KEY_ID: ${{ secrets.APPLE_API_KEY_ID }} From 3b3d24e9e05bc3ddccf55e64559b45dd9c843cb4 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 2 May 2024 19:34:58 +0530 Subject: [PATCH 10/25] It'll need to live in the releases repo, reword accordingly --- .../.github}/workflows/desktop-release.yml | 36 +++++++++++++------ 1 file changed, 25 insertions(+), 11 deletions(-) rename {.github => desktop/.github}/workflows/desktop-release.yml (61%) diff --git a/.github/workflows/desktop-release.yml b/desktop/.github/workflows/desktop-release.yml similarity index 61% rename from .github/workflows/desktop-release.yml rename to desktop/.github/workflows/desktop-release.yml index 60f012b653..cb895fd4de 100644 --- a/.github/workflows/desktop-release.yml +++ b/desktop/.github/workflows/desktop-release.yml @@ -1,17 +1,26 @@ -name: "Release (photos desktop)" +name: "Release" # This will create a new draft release with public artifacts. # # Note that a release will only get created if there is an associated tag # (GitHub releases need a corresponding tag). +# +# The canonical source for this action is in the repository where we keep the +# source code for the Ente Photos desktop app: https://github.com/ente-io/ente +# +# However, it actually lives and runs in the repository that we use for making +# releases: https://github.com/ente-io/photos-desktop +# +# We need two repositories because Electron updater currently doesn't work well +# with monorepos. For more details, see `docs/release.md`. on: - workflow_dispatch: # Allow manually running the action push: - # Run when a tag matching the pattern "photosd-v*"" is pushed - # See: [Note: Testing release workflows that are triggered by tags] + # Run when a tag matching the pattern "v*"" is pushed. + # + # See: [Note: Testing release workflows that are triggered by tags]. tags: - - "photosd-v*" + - "v*" jobs: release: @@ -29,6 +38,11 @@ jobs: - name: Checkout code uses: actions/checkout@v4 with: + # Checkout the tag photosd-v1.x.x from the source code + # repository when we're invoked for tag v1.x.x on the releases + # repository. + repository: ente-io/ente + ref: photosd-${{ github.ref }} submodules: recursive - name: Setup node @@ -44,7 +58,7 @@ jobs: # Import Apple API key for app notarization on macOS run: | mkdir -p ~/private_keys/ - echo '${{ secrets.APPLE_API_KEY }}' > ~/private_keys/AuthKey_${{ secrets.APPLE_API_KEY_ID }}.p8 + echo '${{ secrets.API_KEY }}' > ~/private_keys/AuthKey_${{ secrets.API_KEY_ID }}.p8 - name: Install libarchive-tools for pacman build if: startsWith(matrix.os, 'ubuntu') @@ -60,13 +74,13 @@ jobs: github_token: ${{ secrets.GITHUB_TOKEN }} # If the commit is tagged with a version (e.g. "v1.0.0"), - # release the app after building + # release the app after building. release: ${{ startsWith(github.ref, 'refs/tags/v') }} - mac_certs: ${{ secrets.MAC_OS_CERTIFICATE }} - mac_certs_password: ${{ secrets.MAC_OS_CERTIFICATE_PASSWORD }} + mac_certs: ${{ secrets.MAC_CERTS }} + mac_certs_password: ${{ secrets.MAC_CERTS_PASSWORD }} env: # macOS notarization API key details - API_KEY_ID: ${{ secrets.APPLE_API_KEY_ID }} - API_KEY_ISSUER_ID: ${{ secrets.APPLE_API_KEY_ISSUER_ID }} + API_KEY_ID: ${{ secrets.API_KEY_ID }} + API_KEY_ISSUER_ID: ${{ secrets.API_KEY_ISSUER_ID }} USE_HARD_LINKS: false From fecfb4a8b7d12c4c0540d363bdd2ddad7f60753f Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 2 May 2024 19:52:39 +0530 Subject: [PATCH 11/25] Hopes and dreams --- desktop/docs/release.md | 74 ++++++++++++++++++++++++++--------------- 1 file changed, 47 insertions(+), 27 deletions(-) diff --git a/desktop/docs/release.md b/desktop/docs/release.md index 0a5c2970f5..59b2be10e0 100644 --- a/desktop/docs/release.md +++ b/desktop/docs/release.md @@ -1,33 +1,59 @@ ## Releases -The Github Action that builds the desktop binaries is triggered by pushing a tag -matching the pattern `photosd-v1.2.3`. This value should match the version in -`package.json`. +Conceptually, the release is straightforward: We push a tag, a GitHub workflow +gets triggered that creates a draft release with artifacts built from that tag. +We then publish that release. The download links on our website, and existing +apps already know how to check for the latest GitHub release and update +accordingly. -To make a new release +The complication comes by the fact that Electron Updater (the mechanism that we +use for auto updates) doesn't work well with monorepos. So we need to keep a +separate (non-mono) repository just for doing releases. -1. Create a new branch (can be named anything). On this branch, change the - `version` in `package.json` to `1.x.x` and finalize `CHANGELOG.md`. +- Source code lives here, in [ente-io/ente](https://github.com/ente-io/ente). -2. Commit, tag and push to remote. Note that the tag should have a `photosd-` - prefix: +- Releases are done from + [ente-io/photos-desktop](https://github.com/ente-io/photos-desktop). + +## Workflow + +The workflow is: + +1. Finalize the changes in the source repo. + + - Update the CHANGELOG. + - Update the version in `package.json` + - `git commit -m 'Release v1.x.x'` + - Open PR, merge into main. + + +2. Tag this commit with a tag matching the pattern `photosd-v1.2.3`, where + `1.2.3` is the version in `package.json` ```sh - git add CHANGELOG.md package.json - git commit -m 'Release v1.x.x' git tag photosd-v1.x.x git push origin photosd-v1.x.x ``` - This will trigger the GitHub action that will create a new draft release. +3. Head over to the releases repository, copy all relevant changes from the + source repository, commit and push the changes. -3. To wrap up, increase the version number in `package.json` the next release - train. That is, suppose we just released `v4.0.1`. Then we'll change the - version number in main to `v4.0.2-beta.0`. Each pre-release will modify the - `beta.0` part. Finally, at the time of the next release, this'll become - `v4.0.2`. + ```sh + cp ../ente/desktop/CHANGELOG.md CHANGELOG.md + git add CHANGELOG.md + git commit -m 'Release v1.x.x' + git push origin main + ``` -4. Open a PR for the branch to get it merged into main. +4. Tag this commit, but this time _don't_ use the `photosd-` prefix. Push the + tag to trigger the GitHub action. + + ```sh + git tag v1.x.x + git push origin v1.x.x + ``` + +## Post build The GitHub Action runs on Windows, Linux and macOS. It produces the artifacts defined in the `build` value in `package.json`. @@ -36,17 +62,11 @@ defined in the `build` value in `package.json`. - Linux - An AppImage, and 3 other packages (`.rpm`, `.deb`, `.pacman`) - macOS - A universal DMG -Additionally, the GitHub action notarizes the macOS DMG. For this it needs -credentials provided via GitHub secrets. +Additionally, the GitHub action notarizes and signs the macOS DMG (For this it +uses credentials provided via GitHub secrets). -To rollout the build, we need to publish the draft release. This needs to be -done in the old photos-desktop repository since that the Electron Updater -mechanism doesn't work well with monorepos. So we need to create a new tag with -changelog updates on -[photos-desktop](https://github.com/ente-io/photos-desktop/), use that to create -a new release, copying over all the artifacts. - -Thereafter, everything is automated: +To rollout the build, we need to publish the draft release. Thereafter, +everything is automated: - The website automatically redirects to the latest release on GitHub when people try to download. From 2f2d15c9f2127602e448cbedeb697d777933d134 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 2 May 2024 20:22:40 +0530 Subject: [PATCH 12/25] lint --- desktop/docs/release.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/desktop/docs/release.md b/desktop/docs/release.md index 59b2be10e0..0d1b11bc63 100644 --- a/desktop/docs/release.md +++ b/desktop/docs/release.md @@ -21,11 +21,10 @@ The workflow is: 1. Finalize the changes in the source repo. - - Update the CHANGELOG. - - Update the version in `package.json` - - `git commit -m 'Release v1.x.x'` - - Open PR, merge into main. - + - Update the CHANGELOG. + - Update the version in `package.json` + - `git commit -m 'Release v1.x.x'` + - Open PR, merge into main. 2. Tag this commit with a tag matching the pattern `photosd-v1.2.3`, where `1.2.3` is the version in `package.json` From 67eed1aa89c18a320c5473b74dea6bf68b470332 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 2 May 2024 21:20:26 +0530 Subject: [PATCH 13/25] Upgrade to Electron 30 This picks up the stream fix we need > Fixed data corruption when protocol.handle() processed incoming data asynchronously. #41933 (Also in 31) > > https://github.com/electron/electron/releases/tag/v30.0.2 --- desktop/package.json | 2 +- desktop/yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/desktop/package.json b/desktop/package.json index 5ec8b45be3..d9aaf133ef 100644 --- a/desktop/package.json +++ b/desktop/package.json @@ -43,7 +43,7 @@ "@typescript-eslint/eslint-plugin": "^7", "@typescript-eslint/parser": "^7", "concurrently": "^8", - "electron": "^29", + "electron": "^30", "electron-builder": "^24", "electron-builder-notarize": "^1.5", "eslint": "^8", diff --git a/desktop/yarn.lock b/desktop/yarn.lock index 2210d47450..d4338312bd 100644 --- a/desktop/yarn.lock +++ b/desktop/yarn.lock @@ -1199,10 +1199,10 @@ electron-updater@^6.1: semver "^7.3.8" tiny-typed-emitter "^2.1.0" -electron@^29: - version "29.3.1" - resolved "https://registry.yarnpkg.com/electron/-/electron-29.3.1.tgz#87c82b2cd2c326f78f036499377a5448bea5d4bb" - integrity sha512-auge1/6RVqgUd6TgIq88wKdUCJi2cjESi3jy7d+6X4JzvBGprKBqMJ8JSSFpu/Px1YJrFUKAxfy6SC+TQf1uLw== +electron@^30: + version "30.0.2" + resolved "https://registry.yarnpkg.com/electron/-/electron-30.0.2.tgz#95ba019216bf8be9f3097580123e33ea37497733" + integrity sha512-zv7T+GG89J/hyWVkQsLH4Y/rVEfqJG5M/wOBIGNaDdqd8UV9/YZPdS7CuFeaIj0H9LhCt95xkIQNpYB/3svOkQ== dependencies: "@electron/get" "^2.0.0" "@types/node" "^20.9.0" From 6a990020649965c377bd843a4bbb400e3faaa0fb Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 2 May 2024 21:22:58 +0530 Subject: [PATCH 14/25] Start using it --- desktop/src/main.ts | 8 -------- web/apps/photos/src/utils/native-stream.ts | 23 ++-------------------- 2 files changed, 2 insertions(+), 29 deletions(-) diff --git a/desktop/src/main.ts b/desktop/src/main.ts index eb1114cc4c..49b3162061 100644 --- a/desktop/src/main.ts +++ b/desktop/src/main.ts @@ -127,15 +127,7 @@ const registerPrivilegedSchemes = () => { { scheme: "stream", privileges: { - // TODO(MR): Remove the commented bits if we don't end up - // needing them by the time the IPC refactoring is done. - - // Prevent the insecure origin issues when fetching this - // secure: true, - // Allow the web fetch API in the renderer to use this scheme. supportFetchAPI: true, - // Allow it to be used with video tags. - // stream: true, }, }, ]); diff --git a/web/apps/photos/src/utils/native-stream.ts b/web/apps/photos/src/utils/native-stream.ts index 941c5a9888..4ed9da753a 100644 --- a/web/apps/photos/src/utils/native-stream.ts +++ b/web/apps/photos/src/utils/native-stream.ts @@ -93,40 +93,21 @@ export const writeStream = async ( const params = new URLSearchParams({ path }); const url = new URL(`stream://write?${params.toString()}`); - // TODO(MR): This doesn't currently work. - // - // Not sure what I'm doing wrong here; I've opened an issue upstream - // https://github.com/electron/electron/issues/41872 - // - // A gist with a minimal reproduction - // https://gist.github.com/mnvr/e08d9f4876fb8400b7615347b4d268eb - // - // Meanwhile, write the complete body in one go (this'll eventually run into - // memory failures with large files - just a temporary stopgap to get the - // code to work). - - /* // The duplex parameter needs to be set to 'half' when streaming requests. // // Currently browsers, and specifically in our case, since this code runs // only within our desktop (Electron) app, Chromium, don't support 'full' // duplex mode (i.e. streaming both the request and the response). // https://developer.chrome.com/docs/capabilities/web-apis/fetch-streaming-requests - const req = new Request(`stream://write${path}`, { + const req = new Request(url, { // GET can't have a body method: "POST", body: stream, - // --@ts-expect-error TypeScript's libdom.d.ts does not include the + // @ts-expect-error TypeScript's libdom.d.ts does not include the // "duplex" parameter, e.g. see // https://github.com/node-fetch/node-fetch/issues/1769. duplex: "half", }); - */ - - const req = new Request(url, { - method: "POST", - body: await new Response(stream).blob(), - }); const res = await fetch(req); if (!res.ok) From 0c4da8c86aea2aa00ea47df64a70f367a7410697 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 2 May 2024 21:38:02 +0530 Subject: [PATCH 15/25] POSIX paths --- desktop/src/preload.ts | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/desktop/src/preload.ts b/desktop/src/preload.ts index 589b17fab2..407e541ff7 100644 --- a/desktop/src/preload.ts +++ b/desktop/src/preload.ts @@ -217,7 +217,25 @@ const watchReset = async () => { // - Upload -const pathForFile = (file: File) => webUtils.getPathForFile(file); +const pathForFile = (file: File) => { + const path = webUtils.getPathForFile(file); + // The path that we get back from `webUtils.getPathForFile` on Windows uses + // "/" as the path separator. Convert them to POSIX separators. + // + // Note that we do not have access to the path or the os module in the + // preload script, thus this hand rolled transformation. + + // However that makes TypeScript fidgety since we it cannot find navigator, + // as we haven't included "lib": ["dom"] in our tsconfig to avoid making DOM + // APIs available to our main Node.js code. We could create a separate + // tsconfig just for the preload script, but for now let's go with a cast. + // + // @ts-expect-error navigator is not defined. + const platform = (navigator as { platform: string }).platform; + return platform.toLowerCase().includes("win") + ? path.split("\\").join("/") + : path; +}; const listZipItems = (zipPath: string) => ipcRenderer.invoke("listZipItems", zipPath); From d08c2b4fa0830962d0cdf0b2cb7483d60710c995 Mon Sep 17 00:00:00 2001 From: Crowdin Bot Date: Fri, 3 May 2024 01:40:38 +0000 Subject: [PATCH 16/25] New Crowdin translations by GitHub Action --- .../next/locales/pt-BR/translation.json | 18 +++++++++--------- .../next/locales/zh-CN/translation.json | 4 ++-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/web/packages/next/locales/pt-BR/translation.json b/web/packages/next/locales/pt-BR/translation.json index dfe0030c56..9fc00517cf 100644 --- a/web/packages/next/locales/pt-BR/translation.json +++ b/web/packages/next/locales/pt-BR/translation.json @@ -239,7 +239,7 @@ "ENABLE_MAPS": "Habilitar mapa?", "ENABLE_MAP": "Habilitar mapa", "DISABLE_MAPS": "Desativar Mapas?", - "ENABLE_MAP_DESCRIPTION": "Isto mostrará suas fotos em um mapa do mundo.

Este mapa é hospedado pelo OpenStreetMap , e os exatos locais de suas fotos nunca são compartilhados.

Você pode desativar esse recurso a qualquer momento nas Configurações.

", + "ENABLE_MAP_DESCRIPTION": "

Isto mostrará suas fotos em um mapa do mundo.

Este mapa é hospedado pelo OpenStreetMap, e os exatos locais de suas fotos nunca são compartilhados.

Você pode desativar esse recurso a qualquer momento nas Configurações.

", "DISABLE_MAP_DESCRIPTION": "

Isto irá desativar a exibição de suas fotos em um mapa mundial.

Você pode ativar este recurso a qualquer momento nas Configurações.

", "DISABLE_MAP": "Desabilitar mapa", "DETAILS": "Detalhes", @@ -380,14 +380,14 @@ "LINK_EXPIRED_MESSAGE": "Este link expirou ou foi desativado!", "MANAGE_LINK": "Gerenciar link", "LINK_TOO_MANY_REQUESTS": "Desculpe, este álbum foi visualizado em muitos dispositivos!", - "FILE_DOWNLOAD": "Permitir transferências", + "FILE_DOWNLOAD": "Permitir downloads", "LINK_PASSWORD_LOCK": "Bloqueio de senha", "PUBLIC_COLLECT": "Permitir adicionar fotos", "LINK_DEVICE_LIMIT": "Limite de dispositivos", "NO_DEVICE_LIMIT": "Nenhum", "LINK_EXPIRY": "Expiração do link", "NEVER": "Nunca", - "DISABLE_FILE_DOWNLOAD": "Desabilitar transferência", + "DISABLE_FILE_DOWNLOAD": "Desabilitar download", "DISABLE_FILE_DOWNLOAD_MESSAGE": "

Tem certeza de que deseja desativar o botão de download para arquivos?

Os visualizadores ainda podem capturar imagens da tela ou salvar uma cópia de suas fotos usando ferramentas externas.

", "SHARED_USING": "Compartilhar usando ", "SHARING_REFERRAL_CODE": "Use o código {{referralCode}} para obter 10 GB de graça", @@ -408,8 +408,8 @@ "STOP_ALL_UPLOADS_MESSAGE": "Tem certeza que deseja parar todos os envios em andamento?", "STOP_UPLOADS_HEADER": "Parar envios?", "YES_STOP_UPLOADS": "Sim, parar envios", - "STOP_DOWNLOADS_HEADER": "Parar transferências?", - "YES_STOP_DOWNLOADS": "Sim, parar transferências", + "STOP_DOWNLOADS_HEADER": "Parar downloads?", + "YES_STOP_DOWNLOADS": "Sim, parar downloads", "STOP_ALL_DOWNLOADS_MESSAGE": "Tem certeza que deseja parar todos as transferências em andamento?", "albums_one": "1 Álbum", "albums_other": "{{count, number}} Álbuns", @@ -556,8 +556,8 @@ "SELECT_COLLECTION": "Selecionar álbum", "PIN_ALBUM": "Fixar álbum", "UNPIN_ALBUM": "Desafixar álbum", - "DOWNLOAD_COMPLETE": "Transferência concluída", - "DOWNLOADING_COLLECTION": "Transferindo {{name}}", + "DOWNLOAD_COMPLETE": "Download concluído", + "DOWNLOADING_COLLECTION": "Fazendo download de {{name}}", "DOWNLOAD_FAILED": "Falha ao baixar", "DOWNLOAD_PROGRESS": "{{progress.current}} / {{progress.total}} arquivos", "CHRISTMAS": "Natal", @@ -622,6 +622,6 @@ "TRY_AGAIN": "Tente novamente", "PASSKEY_FOLLOW_THE_STEPS_FROM_YOUR_BROWSER": "Siga os passos do seu navegador para continuar acessando.", "LOGIN_WITH_PASSKEY": "Entrar com a chave de acesso", - "autogenerated_first_album_name": "", - "autogenerated_default_album_name": "" + "autogenerated_first_album_name": "Meu Primeiro Álbum", + "autogenerated_default_album_name": "Novo Álbum" } diff --git a/web/packages/next/locales/zh-CN/translation.json b/web/packages/next/locales/zh-CN/translation.json index d2345f1ae7..c67018aaaf 100644 --- a/web/packages/next/locales/zh-CN/translation.json +++ b/web/packages/next/locales/zh-CN/translation.json @@ -622,6 +622,6 @@ "TRY_AGAIN": "重试", "PASSKEY_FOLLOW_THE_STEPS_FROM_YOUR_BROWSER": "按照浏览器中提示的步骤继续登录。", "LOGIN_WITH_PASSKEY": "使用通行密钥来登录", - "autogenerated_first_album_name": "", - "autogenerated_default_album_name": "" + "autogenerated_first_album_name": "我的第一个相册", + "autogenerated_default_album_name": "新建相册" } From 3eda263d26211418c95a3b2b4a0d1f4b10e06139 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 3 May 2024 09:38:58 +0530 Subject: [PATCH 17/25] Clarify cwd --- docs/docs/self-hosting/guides/custom-server/index.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/docs/self-hosting/guides/custom-server/index.md b/docs/docs/self-hosting/guides/custom-server/index.md index bf695af308..a5ce76cc2b 100644 --- a/docs/docs/self-hosting/guides/custom-server/index.md +++ b/docs/docs/self-hosting/guides/custom-server/index.md @@ -25,10 +25,13 @@ configure the endpoint the app should be connecting to. > You can download the CLI from > [here](https://github.com/ente-io/ente/releases?q=tag%3Acli-v0) -Define a config.yaml and put it either in the same directory as CLI or path -defined in env variable `ENTE_CLI_CONFIG_PATH` +Define a config.yaml and put it either in the same directory as where you run +the CLI from ("current working directory"), or in the path defined in env +variable `ENTE_CLI_CONFIG_PATH`: ```yaml endpoint: api: "http://localhost:8080" ``` + +(Another [example](https://github.com/ente-io/ente/blob/main/cli/config.yaml.example)) From 977d212be6151b54b94946ba3acbc97204a6380d Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 3 May 2024 09:42:17 +0530 Subject: [PATCH 18/25] Add a notice about ente account add --- cli/README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cli/README.md b/cli/README.md index 8fc9aa6948..40858da0f8 100644 --- a/cli/README.md +++ b/cli/README.md @@ -36,7 +36,8 @@ ente --help ### Accounts -If you wish, you can add multiple accounts (your own and that of your family members) and export all data using this tool. +If you wish, you can add multiple accounts (your own and that of your family +members) and export all data using this tool. #### Add an account @@ -44,6 +45,12 @@ If you wish, you can add multiple accounts (your own and that of your family mem ente account add ``` +> [!NOTE] +> +> `ente account add` does not create new accounts, it just adds pre-existing +> accounts to the list of accounts that the CLI knows about so that you can use +> them for other actions. + #### List accounts ```shell From 024f160ca0735f67e3a649058f4bf1fb2290af62 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Fri, 3 May 2024 10:14:27 +0530 Subject: [PATCH 19/25] [mob] Improve log --- mobile/lib/utils/file_uploader.dart | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/mobile/lib/utils/file_uploader.dart b/mobile/lib/utils/file_uploader.dart index d77bc95d7e..f81f9d34bb 100644 --- a/mobile/lib/utils/file_uploader.dart +++ b/mobile/lib/utils/file_uploader.dart @@ -357,10 +357,16 @@ class FileUploader { final List connections = await (Connectivity().checkConnectivity()); bool canUploadUnderCurrentNetworkConditions = true; - if (connections.any((element) => element == ConnectivityResult.mobile)) { - canUploadUnderCurrentNetworkConditions = - Configuration.instance.shouldBackupOverMobileData(); + if (!Configuration.instance.shouldBackupOverMobileData()) { + if (connections.any((element) => element == ConnectivityResult.mobile)) { + canUploadUnderCurrentNetworkConditions = false; + } else { + _logger.info( + "mobileBackupDisabled, backing up with connections: ${connections.map((e) => e.name).toString()}", + ); + } } + if (!canUploadUnderCurrentNetworkConditions) { throw WiFiUnavailableError(); } From ddad863b313b3f9a56d8c2d6a7d00088f0ce47ce Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 3 May 2024 10:50:21 +0530 Subject: [PATCH 20/25] Prepare for release --- desktop/CHANGELOG.md | 8 ++++++++ desktop/package.json | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/desktop/CHANGELOG.md b/desktop/CHANGELOG.md index 83d2123d86..eb118a424d 100644 --- a/desktop/CHANGELOG.md +++ b/desktop/CHANGELOG.md @@ -1,5 +1,13 @@ # CHANGELOG +## v1.7.0 (Unreleased) + +v1.7 is a major rewrite to improve the security of our app. We have enabled +sandboxing and disabled node integration for the renderer process. All this +required restructuring our IPC mechanisms, which resulted in a lot of under the +hood changes. The outcome is a more secure app that also uses the latest and +greatest Electron recommendations. + ## v1.6.63 ### New diff --git a/desktop/package.json b/desktop/package.json index d9aaf133ef..a57219aa35 100644 --- a/desktop/package.json +++ b/desktop/package.json @@ -1,6 +1,6 @@ { "name": "ente", - "version": "1.6.63", + "version": "1.7.0-beta+0", "private": true, "description": "Desktop client for Ente Photos", "author": "Ente ", From 647cc0d80348b1b7944bd903c316d21b173d3a94 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 3 May 2024 11:00:55 +0530 Subject: [PATCH 21/25] [desktop] Fix ref ref in action ref_name is the (from my understanding) the shorthand we need for prefixing. Untested, will do a test build. --- desktop/.github/workflows/desktop-release.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/desktop/.github/workflows/desktop-release.yml b/desktop/.github/workflows/desktop-release.yml index cb895fd4de..ef198ca48a 100644 --- a/desktop/.github/workflows/desktop-release.yml +++ b/desktop/.github/workflows/desktop-release.yml @@ -32,7 +32,9 @@ jobs: strategy: matrix: - os: [macos-latest, ubuntu-latest, windows-latest] + os: [macos-latest] + # Commented for testing + # os: [macos-latest, ubuntu-latest, windows-latest] steps: - name: Checkout code @@ -42,7 +44,7 @@ jobs: # repository when we're invoked for tag v1.x.x on the releases # repository. repository: ente-io/ente - ref: photosd-${{ github.ref }} + ref: photosd-${{ github.ref_name }} submodules: recursive - name: Setup node From 5d0e62cf5fb6d3802d9acc0720e9b9c95587a7df Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 3 May 2024 11:08:06 +0530 Subject: [PATCH 22/25] Use same convention as other preexisting tags in our repo --- .github/workflows/auth-release.yml | 4 ++-- desktop/package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/auth-release.yml b/.github/workflows/auth-release.yml index 707bae895f..174b6c1d33 100644 --- a/.github/workflows/auth-release.yml +++ b/.github/workflows/auth-release.yml @@ -17,8 +17,8 @@ name: "Release (auth)" # We use a suffix like `-test` to indicate that these are test tags, and that # they belong to a pre-release. # -# If you need to do multiple tests, add a +x at the end of the tag. e.g. -# `auth-v1.2.3-test+1`. +# If you need to do multiple tests, add a .x at the end of the tag. e.g. +# `auth-v1.2.3-test.1`. # # Once the testing is done, also delete the tag(s) please. diff --git a/desktop/package.json b/desktop/package.json index a57219aa35..dc5ed9dba4 100644 --- a/desktop/package.json +++ b/desktop/package.json @@ -1,6 +1,6 @@ { "name": "ente", - "version": "1.7.0-beta+0", + "version": "1.7.0-beta.0", "private": true, "description": "Desktop client for Ente Photos", "author": "Ente ", From e9feec37d506772d5a9e2ce8654232cf42e34013 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 3 May 2024 11:28:06 +0530 Subject: [PATCH 23/25] Run the electron builder in the correct path --- desktop/.github/workflows/desktop-release.yml | 2 ++ desktop/docs/release.md | 17 +++-------------- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/desktop/.github/workflows/desktop-release.yml b/desktop/.github/workflows/desktop-release.yml index ef198ca48a..7013d3e579 100644 --- a/desktop/.github/workflows/desktop-release.yml +++ b/desktop/.github/workflows/desktop-release.yml @@ -71,6 +71,8 @@ jobs: - name: Build uses: ente-io/action-electron-builder@v1.0.0 with: + package_root: desktop + # GitHub token, automatically provided to the action # (No need to define this secret in the repo settings) github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/desktop/docs/release.md b/desktop/docs/release.md index 0d1b11bc63..da807b572c 100644 --- a/desktop/docs/release.md +++ b/desktop/docs/release.md @@ -34,22 +34,11 @@ The workflow is: git push origin photosd-v1.x.x ``` -3. Head over to the releases repository, copy all relevant changes from the - source repository, commit and push the changes. +3. Head over to the releases repository and run the trigger script, passing it + the tag _without_ the `photosd-` prefix. ```sh - cp ../ente/desktop/CHANGELOG.md CHANGELOG.md - git add CHANGELOG.md - git commit -m 'Release v1.x.x' - git push origin main - ``` - -4. Tag this commit, but this time _don't_ use the `photosd-` prefix. Push the - tag to trigger the GitHub action. - - ```sh - git tag v1.x.x - git push origin v1.x.x + ./.github/trigger-release.sh v1.x.x ``` ## Post build From b9b928797c7cb141a70358384600a6f7068411c2 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Fri, 3 May 2024 12:24:52 +0530 Subject: [PATCH 24/25] [web][cast] Use server to generate deviceCode --- web/apps/cast/src/pages/index.tsx | 41 ++++++----------------------- web/packages/shared/network/cast.ts | 13 +++++---- 2 files changed, 16 insertions(+), 38 deletions(-) diff --git a/web/apps/cast/src/pages/index.tsx b/web/apps/cast/src/pages/index.tsx index 7ad310fe12..bbba9a1ad7 100644 --- a/web/apps/cast/src/pages/index.tsx +++ b/web/apps/cast/src/pages/index.tsx @@ -9,27 +9,8 @@ import { useEffect, useState } from "react"; import { storeCastData } from "services/cast/castService"; import { useCastReceiver } from "../utils/useCastReceiver"; -// Function to generate cryptographically secure digits -const generateSecureData = (length: number): Uint8Array => { - const array = new Uint8Array(length); - window.crypto.getRandomValues(array); - // Modulo operation to ensure each byte is a single digit - for (let i = 0; i < length; i++) { - array[i] = array[i] % 10; - } - return array; -}; - -const convertDataToDecimalString = (data: Uint8Array): string => { - let decimalString = ""; - for (let i = 0; i < data.length; i++) { - decimalString += data[i].toString(); // No need to pad, as each value is a single digit - } - return decimalString; -}; - export default function PairingMode() { - const [digits, setDigits] = useState([]); + const [deviceCode, setDeviceCode] = useState([]); const [publicKeyB64, setPublicKeyB64] = useState(""); const [privateKeyB64, setPrivateKeyB64] = useState(""); const [codePending, setCodePending] = useState(true); @@ -43,8 +24,6 @@ export default function PairingMode() { const init = async () => { try { - const data = generateSecureData(6); - setDigits(convertDataToDecimalString(data).split("")); const keypair = await generateKeyPair(); setPublicKeyB64(await toB64(keypair.publicKey)); setPrivateKeyB64(await toB64(keypair.privateKey)); @@ -107,7 +86,7 @@ export default function PairingMode() { "urn:x-cast:pair-request", message.senderId, { - code: digits.join(""), + code: deviceCode.join(""), }, ); } catch (e) { @@ -117,9 +96,7 @@ export default function PairingMode() { const generateKeyPair = async () => { await _sodium.ready; - const keypair = _sodium.crypto_box_keypair(); - return keypair; }; @@ -133,7 +110,7 @@ export default function PairingMode() { let devicePayload = ""; try { const encDastData = await castGateway.getCastData( - `${digits.join("")}`, + `${deviceCode.join("")}`, ); if (!encDastData) return; devicePayload = encDastData; @@ -157,10 +134,8 @@ export default function PairingMode() { const advertisePublicKey = async (publicKeyB64: string) => { // hey client, we exist! try { - await castGateway.registerDevice( - `${digits.join("")}`, - publicKeyB64, - ); + const codeValue = await castGateway.registerDevice(publicKeyB64); + setDeviceCode(codeValue.split("")); setCodePending(false); } catch (e) { // schedule re-try after 5 seconds @@ -175,7 +150,7 @@ export default function PairingMode() { useEffect(() => { console.log("useEffect for pairing called"); - if (digits.length < 1 || !publicKeyB64 || !privateKeyB64) return; + if (deviceCode.length < 1 || !publicKeyB64 || !privateKeyB64) return; const interval = setInterval(async () => { console.log("polling for cast data"); @@ -192,7 +167,7 @@ export default function PairingMode() { return () => { clearInterval(interval); }; - }, [digits, publicKeyB64, privateKeyB64, codePending]); + }, [deviceCode, publicKeyB64, privateKeyB64, codePending]); useEffect(() => { if (!publicKeyB64) return; @@ -235,7 +210,7 @@ export default function PairingMode() { ) : ( <> - + )} diff --git a/web/packages/shared/network/cast.ts b/web/packages/shared/network/cast.ts index b240eab32d..a18767baa2 100644 --- a/web/packages/shared/network/cast.ts +++ b/web/packages/shared/network/cast.ts @@ -58,11 +58,14 @@ class CastGateway { return resp.data.publicKey; } - public async registerDevice(code: string, publicKey: string) { - await HTTPService.post(getEndpoint() + "/cast/device-info/", { - deviceCode: `${code}`, - publicKey: publicKey, - }); + public async registerDevice(publicKey: string): Promise { + const resp = await HTTPService.post( + getEndpoint() + "/cast/device-info/", + { + publicKey: publicKey, + }, + ); + return resp.data.deviceCode; } public async publishCastPayload( From 8a859325129d5d4a66f488bb4f1eca151c327952 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Fri, 3 May 2024 12:27:48 +0530 Subject: [PATCH 25/25] refactor --- web/apps/cast/src/pages/index.tsx | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/web/apps/cast/src/pages/index.tsx b/web/apps/cast/src/pages/index.tsx index bbba9a1ad7..b12bf1e765 100644 --- a/web/apps/cast/src/pages/index.tsx +++ b/web/apps/cast/src/pages/index.tsx @@ -10,7 +10,7 @@ import { storeCastData } from "services/cast/castService"; import { useCastReceiver } from "../utils/useCastReceiver"; export default function PairingMode() { - const [deviceCode, setDeviceCode] = useState([]); + const [deviceCode, setDeviceCode] = useState(""); const [publicKeyB64, setPublicKeyB64] = useState(""); const [privateKeyB64, setPrivateKeyB64] = useState(""); const [codePending, setCodePending] = useState(true); @@ -86,7 +86,7 @@ export default function PairingMode() { "urn:x-cast:pair-request", message.senderId, { - code: deviceCode.join(""), + code: deviceCode, }, ); } catch (e) { @@ -109,9 +109,7 @@ export default function PairingMode() { // then, we can decrypt this and store all the necessary info locally so we can play the collection slideshow. let devicePayload = ""; try { - const encDastData = await castGateway.getCastData( - `${deviceCode.join("")}`, - ); + const encDastData = await castGateway.getCastData(`${deviceCode}`); if (!encDastData) return; devicePayload = encDastData; } catch (e) { @@ -135,7 +133,7 @@ export default function PairingMode() { // hey client, we exist! try { const codeValue = await castGateway.registerDevice(publicKeyB64); - setDeviceCode(codeValue.split("")); + setDeviceCode(codeValue); setCodePending(false); } catch (e) { // schedule re-try after 5 seconds @@ -210,7 +208,7 @@ export default function PairingMode() { ) : ( <> - + )}