From b31332ff30e5d5f0cd192f5828868488fdd84b05 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 27 Mar 2024 11:14:06 +0530 Subject: [PATCH 1/3] Outline the plan --- server/scripts/docker.md | 70 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 server/scripts/docker.md diff --git a/server/scripts/docker.md b/server/scripts/docker.md new file mode 100644 index 0000000000..717a503e56 --- /dev/null +++ b/server/scripts/docker.md @@ -0,0 +1,70 @@ +# Running using published Docker images + +Here we describe a way to run an Ente instance using a starter Docker compose +file and using the pre-built Docker images that we publish. This method does not +require you to clone the repository or build any images. + +1. Create a directory where you'll run Ente + + ```sh + mkdir ente && cd ente + ``` + +2. Copy the starter compose.yaml (and two of its support files) from the + repository onto your directory. You can do it by hand, or use (e.g.) curl + + ```sh + # compose.yaml + curl -LO https://raw.githubusercontent.com/ente-io/ente/main/server/compose.yaml + + mkdir -p scripts/compose + cd scripts/compose + + # scripts/compose/credentials.yaml + curl -LO https://raw.githubusercontent.com/ente-io/ente/main/server/scripts/compose/credentials.yaml + + # scripts/compose/minio-provision.sh + curl -LO https://raw.githubusercontent.com/ente-io/ente/main/server/scripts/compose/minio-provision.sh + + cd ../.. + ``` + +3. Modify `compose.yaml`. Instead of asking it to build Ente's server (museum), + we want to get it to pull and use the published Docker image. + + ```diff + --- a/server/compose.yaml + +++ b/server/compose.yaml + @@ -1,9 +1,6 @@ + services: + museum: + - build: + - context: . + - args: + - GIT_COMMIT: development-cluster + + image: ghcr.io/ente-io/server + ``` + +4. Create an (empty) configuration file. Yyou can later put your custom + configuration in this if needed. + + ```sh + touch museum.yaml + ``` + +4. That's it. You can now start everything. + + ```sh + docker compose up + ``` + +This will start a cluster containing: + +* Ente's own server +* PostgresQL (DB) +* MinIO (the S3 layer) + +For each of these, it'll use the latest published Docker image. + +Alternatively, if you only want to run Ente's server, you can directly pull and +run the image we publish to **`ghcr.io/ente-io/server`**. From 7c0f41166c68186ff735925183b1a7f6f39ac41a Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 27 Mar 2024 11:43:19 +0530 Subject: [PATCH 2/3] Add a first cut of the publish script --- .github/workflows/server-publish.yml | 35 ++++++++++++++++++++++++++ .github/workflows/server-release.yml | 8 +++--- server/{scripts => docs}/docker.md | 0 server/docs/publish.md | 37 ++++++++++++++++++++++++++++ 4 files changed, 76 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/server-publish.yml rename server/{scripts => docs}/docker.md (100%) create mode 100644 server/docs/publish.md diff --git a/.github/workflows/server-publish.yml b/.github/workflows/server-publish.yml new file mode 100644 index 0000000000..1d81484efc --- /dev/null +++ b/.github/workflows/server-publish.yml @@ -0,0 +1,35 @@ +name: "Publish (server)" + +on: + # Run manually, providing it the commit. + # + # To obtain the commit from the currently deployed museum, do: + # curl -s https://api.ente.io/ping | jq -r '.id' + workflow_dispatch: + inputs: + commit: + description: "Commit to publish the image from" + type: string + required: true + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout code @ ${{ inputs.commit }} + uses: actions/checkout@v4 + with: + ref: ${{ inputs.commit }} + + - name: Build and push + uses: mr-smithers-excellent/docker-build-push@v6 + with: + dockerfile: server/Dockerfile + directory: server + image: ghcr.io/ente-io/server + registry: ghcr.io + enableBuildKit: true + buildArgs: GIT_COMMIT=${GITHUB_SHA} + tags: ${GITHUB_SHA}, latest + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/server-release.yml b/.github/workflows/server-release.yml index 8f0281951a..fa02155300 100644 --- a/.github/workflows/server-release.yml +++ b/.github/workflows/server-release.yml @@ -7,11 +7,11 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - name: Check out code + - name: Checkout code + uses: actions/checkout@v4 - - uses: mr-smithers-excellent/docker-build-push@v6 - name: Build & Push + - name: Build and push + uses: mr-smithers-excellent/docker-build-push@v6 with: dockerfile: server/Dockerfile directory: server diff --git a/server/scripts/docker.md b/server/docs/docker.md similarity index 100% rename from server/scripts/docker.md rename to server/docs/docker.md diff --git a/server/docs/publish.md b/server/docs/publish.md new file mode 100644 index 0000000000..5f7be8e447 --- /dev/null +++ b/server/docs/publish.md @@ -0,0 +1,37 @@ +# Publishing images + +There are two different images we publish - internal and external. + +## Internal + +The internal images can be built and run by triggering the "Server (release)" +workflow. You can trigger it either from GitHub's UI on the Actions tab, or use +the following command: + + gh workflow run server-release.yml + +This will take the latest main, package it into a Docker image, and publish it +to our Scaleway registry. From there, we can update our production instances to +use this new image (see [deploy/README](../scripts/deploy/README.md)). + +## External + +Periodically, we can republish a new image from an existing known-to-be-good +commit to the GitHub Container Registry (GHCR) so that it can be used by folks +without needing to clone our repository just for building an image. For more +details about the use case, see [docker.md](docker.md). + +To publish such an external image, firstly find the commit of the currently +running production instance. + + curl -s https://api.ente.io/ping | jq -r '.id' + +> We can publish from any arbitrary commit really, but by using the commit +> that's already seen production for a few days, we avoid externally publishing +> images with issues. + +Then, trigger the "Publish (server)" workflow, providing it the commit. You can +trigger it either from GitHub's UI or using the `gh cli`. With the CLI, we can +combine both these steps too: + +Once the workflow completes, the resultant image will be available at From 4ba76e4aa2591556612f5fbf960d21800bc2e627 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 27 Mar 2024 12:46:08 +0530 Subject: [PATCH 3/3] Tweaks --- server/docs/docker.md | 6 +++--- server/docs/publish.md | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/server/docs/docker.md b/server/docs/docker.md index 717a503e56..a6e0e2cd88 100644 --- a/server/docs/docker.md +++ b/server/docs/docker.md @@ -29,8 +29,8 @@ require you to clone the repository or build any images. cd ../.. ``` -3. Modify `compose.yaml`. Instead of asking it to build Ente's server (museum), - we want to get it to pull and use the published Docker image. +3. Modify `compose.yaml`. Instead of building from source, we want directly use + the published Docker image from `ghcr.io/ente-io/server` ```diff --- a/server/compose.yaml @@ -52,7 +52,7 @@ require you to clone the repository or build any images. touch museum.yaml ``` -4. That's it. You can now start everything. +4. That is all. You can now start everything. ```sh docker compose up diff --git a/server/docs/publish.md b/server/docs/publish.md index 5f7be8e447..7984648e4a 100644 --- a/server/docs/publish.md +++ b/server/docs/publish.md @@ -32,6 +32,6 @@ running production instance. Then, trigger the "Publish (server)" workflow, providing it the commit. You can trigger it either from GitHub's UI or using the `gh cli`. With the CLI, we can -combine both these steps too: +combine both these steps too. -Once the workflow completes, the resultant image will be available at +Once the workflow completes, the resultant image will be available at `ghcr.io/ente-io/server`.