diff --git a/docs/docs/.vitepress/sidebar.ts b/docs/docs/.vitepress/sidebar.ts index 5148296f11..06f64d790b 100644 --- a/docs/docs/.vitepress/sidebar.ts +++ b/docs/docs/.vitepress/sidebar.ts @@ -267,7 +267,7 @@ export const sidebar = [ link: "/self-hosting/install/without-docker", }, { - text: "Environment Variable and Defaults", + text: "Environment Variables and Defaults", link: "/self-hosting/install/env-var", }, { @@ -275,7 +275,7 @@ export const sidebar = [ link: "/self-hosting/install/config-file", }, { - text: "Post Installation", + text: "Post-installation Steps", link: "/self-hosting/install/post-install/", } ], @@ -289,8 +289,8 @@ export const sidebar = [ link: "/self-hosting/administration/museum", }, { - text: "Configuring S3", - link: "/self-hosting/administration/configuring-s3", + text: "Configuring Object Storage", + link: "/self-hosting/administration/object-storage", }, { text: "Reverse proxy", @@ -313,6 +313,16 @@ export const sidebar = [ } ] }, + { + text: "Community Guides", + collapsed: true, + items: [ + { + text: "Ente via Tailscale", + link: "/self-hosting/guides/tailscale", + }, + ], + }, { text: "Troubleshooting", collapsed: true, @@ -321,10 +331,6 @@ export const sidebar = [ text: "General", link: "/self-hosting/troubleshooting/misc", }, - { - text: "Bucket CORS", - link: "/self-hosting/troubleshooting/bucket-cors", - }, { text: "Uploads", link: "/self-hosting/troubleshooting/uploads", @@ -335,24 +341,6 @@ export const sidebar = [ }, ], }, - { - text: "Community Guides", - collapsed: true, - items: [ - { - text: "Ente via Tailscale", - link: "/self-hosting/guides/tailscale", - }, - { - text: "Ente with External S3", - link: "/self-hosting/guides/external-s3", - }, - ], - }, - { - text: "FAQ", - link: "/self-hosting/faq/", - } ], }, ]; diff --git a/docs/docs/self-hosting/install/defaults.md b/docs/docs/self-hosting/administration/accounts.md similarity index 100% rename from docs/docs/self-hosting/install/defaults.md rename to docs/docs/self-hosting/administration/accounts.md diff --git a/docs/docs/self-hosting/administration/configuring-s3.md b/docs/docs/self-hosting/administration/configuring-s3.md deleted file mode 100644 index e9cb3ebb26..0000000000 --- a/docs/docs/self-hosting/administration/configuring-s3.md +++ /dev/null @@ -1,45 +0,0 @@ ---- -title: Configuring S3 buckets -description: - Configure S3 endpoints to fix upload errors or use your self hosted ente - from outside localhost ---- - -# Configuring S3 - -## Replication - -> [!IMPORTANT] -> -> As of now, replication works only if all the 3 storage buckets are configured (2 hot and 1 cold storage). -> -> For more information, check this -> [discussion](https://github.com/ente-io/ente/discussions/3167#discussioncomment-10585970) -> and our article on ensuring [reliability](https://ente.io/reliability/). - -In a self hosted Ente instance replication is turned off by default. When -replication is turned off, only the first bucket (`b2-eu-cen`) is used, and the -other two are ignored. Only the names here are specifically fixed, but in the -configuration body you can put any other keys. - -Use the `s3.hot_storage.primary` option if you'd like to set one of the other -predefined buckets as the primary bucket. - -## SSL Configuration - -> [!NOTE] -> -> If you need to configure SSL, you'll need to turn off `s3.are_local_buckets` -> (which disables SSL in the default starter compose template). - -Disabling `s3.are_local_buckets` also switches to the subdomain style URLs for -the buckets. However, not all S3 providers support these. In particular, MinIO -does not work with these in default configuration. So in such cases you'll also -need to enable `s3.use_path_style_urls`. - -## Summary - -Set the S3 bucket `endpoint` in `credentials.yaml` to a `yourserverip:3200` or -some such IP / hostname that is accessible from both where you are running the -Ente clients (e.g. the mobile app) and also from within the Docker compose -cluster. \ No newline at end of file diff --git a/docs/docs/self-hosting/administration/museum.md b/docs/docs/self-hosting/administration/museum.md index 998e7ad4b6..3837bbad1b 100644 --- a/docs/docs/self-hosting/administration/museum.md +++ b/docs/docs/self-hosting/administration/museum.md @@ -53,7 +53,6 @@ apps: public-albums: https://albums.myente.xyz cast: https://cast.myente.xyz accounts: https://accounts.myente.xyz - family: https://family.myente.xyz ``` > [!IMPORTANT] By default, all the values redirect to our publicly hosted @@ -70,4 +69,4 @@ stop all the Docker containers with `docker compose down` and restart them with Similarly, you can use the default [`local.yaml`](https://github.com/ente-io/ente/tree/main/server/configurations/local.yaml) as a reference for building a functioning `museum.yaml` for many other -functionalities like SMTP, Discord notifications, Hardcoded-OTTs, etc. \ No newline at end of file +functionalities like SMTP, Hardcoded-OTTs, etc. \ No newline at end of file diff --git a/docs/docs/self-hosting/administration/object-storage.md b/docs/docs/self-hosting/administration/object-storage.md new file mode 100644 index 0000000000..ffb7d6bc0a --- /dev/null +++ b/docs/docs/self-hosting/administration/object-storage.md @@ -0,0 +1,85 @@ +--- +title: Configuring Object Storage +description: + Configure Object Storage for storing files along with some troubleshooting tips +--- + +# Configuring Object Storage + +## Replication + +> [!IMPORTANT] +> +> As of now, replication works only if all the 3 storage buckets are configured (2 hot and 1 cold storage). +> +> For more information, check this +> [discussion](https://github.com/ente-io/ente/discussions/3167#discussioncomment-10585970) +> and our article on ensuring [reliability](https://ente.io/reliability/). + +In a self hosted Ente instance replication is turned off by default. When +replication is turned off, only the first bucket (`b2-eu-cen`) is used, and the +other two are ignored. Only the names here are specifically fixed, but in the +configuration body you can put any other keys. + +Use the `s3.hot_storage.primary` option if you'd like to set one of the other +predefined buckets as the primary bucket. + +## SSL Configuration + +> [!NOTE] +> +> If you need to configure SSL, you'll need to turn off `s3.are_local_buckets` +> (which disables SSL in the default starter compose template). + +Disabling `s3.are_local_buckets` also switches to the subdomain style URLs for +the buckets. However, not all S3 providers support these. In particular, MinIO +does not work with these in default configuration. So in such cases you'll also +need to enable `s3.use_path_style_urls`. + +# Fix potential CORS issues with your Buckets + +## For AWS S3 + +If you cannot upload a photo due to a CORS issue, you need to fix the CORS +configuration of your bucket. + +Create a `cors.json` file with the following content: + +```json +{ + "CORSRules": [ + { + "AllowedOrigins": ["*"], + "AllowedHeaders": ["*"], + "AllowedMethods": ["GET", "HEAD", "POST", "PUT", "DELETE"], + "MaxAgeSeconds": 3000, + "ExposeHeaders": ["Etag"] + } + ] +} +``` + +You may want to change the `AllowedOrigins` to a more restrictive value. + +If you are using AWS for S3, you can execute the below command to get rid of +CORS. Make sure to enter the right path for the `cors.json` file. + +```bash +aws s3api put-bucket-cors --bucket YOUR_S3_BUCKET --cors-configuration /path/to/cors.json +``` + +## For MinIO + +Checkout the `mc set alias` document to configure alias for your +instance and bucket. After this you will be prompted for your AccessKey and +Secret, which is your username and password. + +To set the `AllowedOrigins` Header, you can use the +following command to do so. + +```sh +mc admin config set / api cors_allow_origin="*" +``` + +You can create also `.csv` file and dump the list of origins you would like to +allow and replace the `*` with `path` to the CSV file. diff --git a/docs/docs/self-hosting/administration/reverse-proxy.md b/docs/docs/self-hosting/administration/reverse-proxy.md index 97ef7da570..5c931d7cbb 100644 --- a/docs/docs/self-hosting/administration/reverse-proxy.md +++ b/docs/docs/self-hosting/administration/reverse-proxy.md @@ -1,53 +1,102 @@ --- Title: Configuring Reverse Proxy -Description: configuring reverse proxy for Museum and other endpoints +Description: Configuring reverse proxy for Museum and other services --- # Reverse proxy -Ente's server (museum) runs on port `:8080`, web app on `:3000` and the other -apps from ports `3001-3004`. - -We highly recommend using HTTPS for Museum (`8080`). For security reasons museum +We highly recommend using HTTPS for Museum (`8080`). For security reasons, Museum will not accept incoming HTTP traffic. -Head over to your DNS management dashboard and setup the appropriate records for -the endpoints. Mostly, `A` or `AAAA` records targeting towards your server's IP -address should be sufficient. The rest of the work will be done by the web -server on your machine. + +## Pre-requisites + +1. **Reverse Proxy:** We recommend using Caddy for simplicity of +configuration and automatic certificate generation and management, +although you can use other alternatives such as NGINX, Traefik, etc. + + Install Caddy using the following command on Debian/Ubuntu-based systems: + ``` shell + sudo apt install caddy + ``` + + Start the service, enable it to start upon system boot and reload when configuration + has changed. + + ``` shell + sudo systemctl start caddy + + sudo systemctl enable caddy + + sudo systemctl reload caddy + ``` + +## Step 1: Configure A or AAAA records + +Set up the appropriate records for the endpoints in your DNS +management dashboard (usually associated with your domain registrar). + +`A` or `AAAA` records targeting towards your server's IP address should +be sufficient. ![cloudflare](/cloudflare.png) -### Caddy +## Step 2: Configure reverse proxy -Setting up a reverse proxy with Caddy is easy and straightforward. +Once Caddy is installed on system, a `Caddyfile` is created on the path +`/etc/caddy/`. Edit `/etc/caddy/Caddyfile` to configure reverse proxies. -Firstly, install Caddy on your server. - -```sh -sudo apt install caddy -``` - -After the installation is complete, a `Caddyfile` is created on the path -`/etc/caddy/`. This file is used to configure reverse proxies among other -things. +Here is a ready-to-use configuration that can be used with your own domain. ```groovy -# Caddyfile - myente.xyz is just an example. +# yourdomain.tld is an example. Replace it with your own domain -api.myente.xyz { +# For Museum +api.ente.yourdomain.tld { reverse_proxy http://localhost:8080 } -ente.myente.xyz { +# For Ente Photos web app +web.ente.yourdomain.tld { reverse_proxy http://localhost:3000 } -#...and so on for other endpoints +# For Ente Accounts web app +accounts.ente.yourdomain.tld { + reverse_proxy http://localhost:3001 +} + +# For Ente Albums web app +albums.ente.yourdomain.tld { + reverse_proxy http://localhost:3002 +} + +# For Ente Auth web app +auth.ente.yourdomain.tld { + reverse_proxy http://localhost:3003 +} + +# For Ente Cast web app +cast.ente.yourdomain.tld { + reverse_proxy http://localhost:3004 +} + +# For Museum +api.ente.yourdomain.tld { + reverse_proxy http://localhost:8080 +} ``` -After a hard-reload, the Ente Photos web app should be up on -https://ente.myente.xyz. +## Step 3: Start reverse proxy -If you are using a different tool for reverse proxy (like nginx), please check -out their documentation. +Reload Caddy for changes to take effect + +``` shell +sudo systemctl caddy reload +``` + +Ente Photos web app should be up on https://web.ente.yourdomain.tld. + +> [!TIP] +> If you are using other reverse proxy servers such as NGINX, +> Traefik, etc., please check out their documentation. diff --git a/docs/docs/self-hosting/administration/security.md b/docs/docs/self-hosting/administration/security.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/docs/self-hosting/administration/selfhost-cli.md b/docs/docs/self-hosting/administration/selfhost-cli.md index 9e5d312290..f5ff00bee6 100644 --- a/docs/docs/self-hosting/administration/selfhost-cli.md +++ b/docs/docs/self-hosting/administration/selfhost-cli.md @@ -23,8 +23,8 @@ and subsequently increase the [storage and account validity](https://github.com/ente-io/ente/blob/main/cli/docs/generated/ente_admin_update-subscription.md) using the CLI. -For the admin actions, you first need to whitelist admin users. You can create -`server/museum.yaml`, and whitelist add the admin userID `internal.admins`. See +For administrative actions, you first need to whitelist admin users. +You can create `server/museum.yaml`, and whitelist add the admin userID `internal.admins`. See [local.yaml](https://github.com/ente-io/ente/blob/main/server/configurations/local.yaml#L211C1-L232C1) in the server source code for details about how to define this. diff --git a/docs/docs/self-hosting/faq/index.md b/docs/docs/self-hosting/faq/index.md deleted file mode 100644 index eedd1fbd86..0000000000 --- a/docs/docs/self-hosting/faq/index.md +++ /dev/null @@ -1,47 +0,0 @@ ---- -title: FAQ - Self hosting -description: Frequently asked questions about self hosting Ente ---- - -# Frequently Asked Questions - -### Do Ente Photos and Ente Auth share the same backend? - -Yes. The apps share the same backend, the same database and the same object -storage namespace. The same user account works for both of them. - -### Can I just self host Ente Auth? - -Yes, if you wish, you can self-host the server and use it only for the 2FA auth -app. The starter Docker compose will work fine for either Photos or Auth (or -both!). - -> You currently don't need to configure the S3 object storage (e.g. minio -> containers) if you're only using your self hosted Ente instance for auth. - -### Can I use the server with _X_ as the object storage? - -Yes. As long as whatever X you're using provides an S3 compatible API, you can -use it as the underlying object storage. For example, the starter self-hosting -Docker compose file we offer uses MinIO, and on our production deployments we -use Backblaze/Wasabi/Scaleway. But that's not the full list - as long as the -service you intend to use has a S3 compatible API, it can be used. - -### How do I increase storage space for users on my self hosted instance? - -See the [guide for administering your server](/self-hosting/guides/admin). In -particular, you can use the `ente admin update-subscription` CLI command to -increase the -[storage and account validity](https://github.com/ente-io/ente/blob/main/cli/docs/generated/ente_admin_update-subscription.md) -of accounts on your instance. - -### How can I become an admin on my self hosted instance? - -The first user you create on your instance is treated as an admin. - -If you want, you can modify this behaviour by providing an explicit list of -admins in the [configuration](/self-hosting/guides/admin#becoming-an-admin). - -### Can I disable registration of new accounts on my self hosted instance? - -Yes. See `internal.disable-registration` in local.yaml. diff --git a/docs/docs/self-hosting/guides/systemd.md b/docs/docs/self-hosting/guides/systemd.md new file mode 100644 index 0000000000..ac6e0e4296 --- /dev/null +++ b/docs/docs/self-hosting/guides/systemd.md @@ -0,0 +1,28 @@ +--- +title: Running Ente using systemd - Self-hosting +description: Running Ente services (Museum and web application) via systemd +--- + +# Running Ente using `systemd` + +On Linux distributions using `systemd` as initialization system, Ente can be configured to run as a background service, upon system startup by using service files. + +## Museum as a background service + +Please check the below links if you want to run Museum as a service, both of +them are battle tested. + +1. [How to run museum as a systemd service](https://gist.github.com/mngshm/a0edb097c91d1dc45aeed755af310323) +2. [Museum.service](https://github.com/ente-io/ente/blob/23e678889189157ecc389c258267685934b83631/server/scripts/deploy/museum.service#L4) + +Once you are done with setting and running Museum, all you are left to do is run +the web app and reverse_proxy it with a webserver. You can check the following + resources for Deploying your web app. + +1. [Running Ente Web app as a systemd Service](https://gist.github.com/mngshm/72e32bd483c2129621ed0d74412492fd) + +## References + +1. [How to run museum as a systemd service](https://gist.github.com/mngshm/a0edb097c91d1dc45aeed755af310323) +2. [Museum.service](https://github.com/ente-io/ente/blob/23e678889189157ecc389c258267685934b83631/server/scripts/deploy/museum.service#L4) +3. [Running Ente Web app as a systemd Service](https://gist.github.com/mngshm/72e32bd483c2129621ed0d74412492fd) diff --git a/docs/docs/self-hosting/index.md b/docs/docs/self-hosting/index.md index 4c170fe150..e8ff85a6a1 100644 --- a/docs/docs/self-hosting/index.md +++ b/docs/docs/self-hosting/index.md @@ -6,7 +6,7 @@ description: Getting started with self-hosting Ente # Get Started If you're looking to spin up Ente on your server for preserving those -sweet memories, you are in the right place! +sweet memories or using Auth for 2FA codes, you are in the right place! Our entire source code ([including the server](https://ente.io/blog/open-sourcing-our-server/)) is open source. This is the same code we use on production. @@ -97,7 +97,6 @@ You can import your pictures from Google Takeout or from other services to Ente You can import your codes from other authenticator providers to Ente Auth. Check out the [migration guide](/auth/migration/) for more information. - ## Queries? If you need support, please ask on our community diff --git a/docs/docs/self-hosting/install/compose.md b/docs/docs/self-hosting/install/compose.md index 4ac1cd724b..19e96875f6 100644 --- a/docs/docs/self-hosting/install/compose.md +++ b/docs/docs/self-hosting/install/compose.md @@ -7,9 +7,14 @@ description: Running Ente with Docker Compose from source If you wish to run Ente via Docker Compose from source, do the following: +## Requirements + +Check out the [requirements](/self-hosting/install/requirements) page to get +started. + ## Step 1: Clone the repository -Clone the repository to a prefered directory. Change into the `server/config` directory inside the cloned repository, where the compose file for running the cluster is present. +Clone the repository. Change into the `server/config` directory of the repository, where the Compose file for running the cluster is present. Run the following command for the same: @@ -37,10 +42,14 @@ Change the values present in `.env` file along with `museum.yaml` file according ## Step 3: Start the cluster -Now you can start the cluster by running the following command: +Start the cluster by running the following command: ```sh docker compose up --build ``` This builds Museum and web applications based on the Dockerfile and starts the containers needed for Ente. + +::: tip +Check out [post-installation steps](/self-hosting/install/post-install) for further usage. +::: \ No newline at end of file diff --git a/docs/docs/self-hosting/install/config-file.md b/docs/docs/self-hosting/install/config-file.md index 975b25772c..5b48c93a31 100644 --- a/docs/docs/self-hosting/install/config-file.md +++ b/docs/docs/self-hosting/install/config-file.md @@ -5,10 +5,132 @@ description: museum.yaml" --- - # Configuration File -Ente's server, Museum, uses YAML-based configuration file that is used for handling database -connectivity, bucket configuration, admin user whitelisting, etc. +Museum uses YAML-based configuration file that is used for handling database +connectivity, bucket configuration, internal configuration, etc. -Self-hosted clusters generally use `museum.yaml` file for reading configuration specifications. +If the `ENVIRONMENT` environment variable is set, a corresponding file from +`configurations/` is loaded, by default, `local.yaml` configuration is loaded. + +If `credentials-file` is defined and found, it overrides the defaults. + +Self-hosted clusters generally use `museum.yaml` file for reading configuration +specifications. + +All configuration values can be overridden via environment variables using the +`ENTE_` prefix and replacing dots (`.`) or hyphens (`-`) with underscores (`_`). + +The configuration variables declared in `museum.yaml` are read by Museum. +Additionally, environment variables prefixed by `ENTE_` are read by Museum and +used internally in same manner as configuration variables. + +For example, `s3.b2-eu-cen` in `museum.yaml` and `ENTE_S3_B2_EU_CEN` declared as +environment variable are the same and `ENTE_S3_B2_EU_CEN` overrides +`s3.b2-eu-cen`. + +## General Settings + +| Variable | Description | Default | +| ------------------ | --------------------------------------------------------- | ------------------ | +| `credentials-file` | Path to optional credentials override file | `credentials.yaml` | +| `credentials-dir` | Directory to look for credentials (TLS, service accounts) | `credentials/` | +| `log-file` | Log output path. Required in production. | `""` | + +## HTTP + +| Variable | Description | Default | +| -------------- | --------------------------------- | ------- | +| `http.use-tls` | Enables TLS and binds to port 443 | `false` | + +## App Endpoints + +| Variable | Description | Default | +| -------------------- | ------------------------------------------------------- | -------------------------- | +| `apps.public-albums` | Albums app base endpoint for public sharing | `https://albums.ente.io` | +| `apps.cast` | Cast app base endpoint | `https://cast.ente.io` | +| `apps.accounts` | Accounts app base endpoint (used for passkey-based 2FA) | `https://accounts.ente.io` | + +## Database + +| Variable | Description | Default | +| ------------- | -------------------------- | ----------- | +| `db.host` | DB hostname | `localhost` | +| `db.port` | DB port | `5432` | +| `db.name` | Database name | `ente_db` | +| `db.sslmode` | SSL mode for DB connection | `disable` | +| `db.user` | Database username | | +| `db.password` | Database password | | +| `db.extra` | Additional DSN parameters | | + +## Object Storage + +| Variable | Description | Default | +| -------------------------------------- | -------------------------------------------- | ------- | +| `s3.b2-eu-cen` | Primary hot storage S3 config | | +| `s3.wasabi-eu-central-2-v3.compliance` | Whether to disable compliance lock on delete | `true` | +| `s3.scw-eu-fr-v3` | Optional secondary S3 config | | +| `s3.wasabi-eu-central-2-derived` | Derived data storage | | +| `s3.are_local_buckets` | Use local MinIO-compatible storage | `false` | +| `s3.use_path_style_urls` | Enable path-style URLs for MinIO | `false` | + +## Encryption Keys + +| Variable | Description | Default | +| ---------------- | -------------------------------------- | ----------- | +| `key.encryption` | Key for encrypting user emails | Pre-defined | +| `key.hash` | Hash key for verifying email integrity | Pre-defined | + +## JWT + +| Variable | Description | Default | +| ------------ | ----------------------- | ---------- | +| `jwt.secret` | Secret for signing JWTs | Predefined | + +## Email + +| Variable | Description | Default | +| ------------------ | ---------------------------- | ------- | +| `smtp.host` | SMTP server host | | +| `smtp.port` | SMTP server port | | +| `smtp.username` | SMTP auth username | | +| `smtp.password` | SMTP auth password | | +| `smtp.email` | Sender email address | | +| `smtp.sender-name` | Custom name for email sender | | +| `transmail.key` | Zeptomail API key | | + +## WebAuthn Passkey Support + +| Variable | Description | Default | +| -------------------- | ---------------------------- | --------------------------- | +| `webauthn.rpid` | Relying Party ID | `localhost` | +| `webauthn.rporigins` | Allowed origins for WebAuthn | `["http://localhost:3001"]` | + +## Internal + +| Variable | Description | Default | +| ------------------------------- | --------------------------------------------- | ------- | +| `internal.silent` | Suppress external effects (e.g. email alerts) | `false` | +| `internal.health-check-url` | External healthcheck URL | | +| `internal.hardcoded-ott` | Predefined OTPs for testing | | +| `internal.admins` | List of admin user IDs | `[]` | +| `internal.admin` | Single admin user ID | | +| `internal.disable-registration` | Disable user registration | `false` | + +## Replication + +| Variable | Description | Default | +| -------------------------- | ------------------------------------ | ----------------- | +| `replication.enabled` | Enable cross-datacenter replication | `false` | +| `replication.worker-url` | Cloudflare Worker for replication | | +| `replication.worker-count` | Number of goroutines for replication | `6` | +| `replication.tmp-storage` | Temp directory for replication | `tmp/replication` | + +## Background Jobs + +| Variable | Description | Default | +| --------------------------------------------- | --------------------------------------- | ------- | +| `jobs.cron.skip` | Skip all cron jobs | `false` | +| `jobs.remove-unreported-objects.worker-count` | Workers for removing unreported objects | `1` | +| `jobs.clear-orphan-objects.enabled` | Enable orphan cleanup | `false` | +| `jobs.clear-orphan-objects.prefix` | Prefix filter for orphaned objects | | diff --git a/docs/docs/self-hosting/install/env-var.md b/docs/docs/self-hosting/install/env-var.md index b79a53f6d7..ee60836ab4 100644 --- a/docs/docs/self-hosting/install/env-var.md +++ b/docs/docs/self-hosting/install/env-var.md @@ -7,8 +7,7 @@ description: # Environment Variables and Defaults -The environment variables needed for running Ente, configuration variables -present in Museum's configuration file and the default configuration are +The environment variables needed for running Ente and the default configuration are documented below: ## Environment Variables @@ -19,15 +18,15 @@ and port mappings of the web apps. Here's the list of environment variables that is used by the cluster: -| Service | Environment Variable | Description | Default Value | -| ---------- | --------------------- | ------------------------------------------------ | --------------------------------- | -| `web` | `ENTE_API_ORIGIN` | API Endpoint for Ente's API (Museum) | http://localhost:8080 | -| `web` | `ENTE_ALBUMS_ORIGIN` | Base URL for Ente Album, used for public sharing | http://localhost:3002 | -| `postgres` | `POSTGRES_USER` | Username for PostgreSQL database | pguser | -| `postgres` | `POSTGRES_DB` | Name of database for use with Ente | ente_db | -| `postgres` | `POSTGRES_PASSWORD` | Password for PostgreSQL database's user | Randomly generated for quickstart | -| `minio` | `MINIO_ROOT_USER` | Username for MinIO | Randomly generated for quickstart | -| `minio` | `MINIO_ROOT_PASSWORD` | Password for MinIO | Randomly generated for quickstart | +| Service | Environment Variable | Description | Default Value | +| ---------- | --------------------- | ----------------------------------------------------------------------------------------------- | ------------------------------- | +| `web` | `ENTE_API_ORIGIN` | Alias for `NEXT_PUBLIC_ENTE_ENDPOINT`. API Endpoint for Ente's API (Museum). | http://localhost:8080 | +| `web` | `ENTE_ALBUMS_ORIGIN` | Alias for `NEXT_PUBLIC_ENTE_ALBUMS_ENDPOINT`. Base URL for Ente Album, used for public sharing. | http://localhost:3002 | +| `postgres` | `POSTGRES_USER` | Username for PostgreSQL database | pguser | +| `postgres` | `POSTGRES_DB` | Name of database for use with Ente | ente_db | +| `postgres` | `POSTGRES_PASSWORD` | Password for PostgreSQL database's user | Randomly generated (quickstart) | +| `minio` | `MINIO_ROOT_USER` | Username for MinIO | Randomly generated (quickstart) | +| `minio` | `MINIO_ROOT_PASSWORD` | Password for MinIO | Randomly generated (quickstart) | ## Default Configuration @@ -37,16 +36,16 @@ which is documented below to understand its behavior: ### Ports The below format is according to how ports are mapped in Docker when using -quickstart script. The mapping is of the format `- :` +quickstart script. The mapping is of the format `:` in `ports` in compose file. -| Service | Type | Host Port | Container Port | -| ---------------------------------- | -------- | --------- | -------------- | -| Museum | Server | 8080 | 8080 | -| Ente Photos | Web | 3000 | 3000 | -| Ente Accounts | Web | 3001 | 3001 | -| Ente Albums | Web | 3002 | 3002 | -| [Ente Auth](https://ente.io/auth/) | Web | 3003 | 3003 | -| [Ente Cast](http://ente.io/cast) | Web | 3004 | 3004 | -| MinIO | S3 | 3200 | 3200 | -| PostgreSQL | Database | | 5432 | +| Service | Type | Host Port | Container Port | +| ------------------------------------------------------- | -------- | --------- | -------------- | +| Museum | Server | 8080 | 8080 | +| Ente Photos | Web | 3000 | 3000 | +| Ente Accounts | Web | 3001 | 3001 | +| Ente Albums | Web | 3002 | 3002 | +| [Ente Auth](https://ente.io/auth/) | Web | 3003 | 3003 | +| [Ente Cast](https://help.ente.io/photos/features/cast/) | Web | 3004 | 3004 | +| MinIO | S3 | 3200 | 3200 | +| PostgreSQL | Database | | 5432 | diff --git a/docs/docs/self-hosting/install/post-install/index.md b/docs/docs/self-hosting/install/post-install/index.md index a3d1d4f703..32eabd0072 100644 --- a/docs/docs/self-hosting/install/post-install/index.md +++ b/docs/docs/self-hosting/install/post-install/index.md @@ -9,7 +9,7 @@ A list of steps that should be done after installing Ente are described below: ## Step 1: Creating first user -The first user to be created will be treated as an admin user. +The first user (and the only user) to be created will be treated as an admin user by default. Once Ente is up and running, the Ente Photos web app will be accessible on `http://localhost:3000`. Open this URL in your browser and proceed with creating @@ -28,10 +28,11 @@ sudo docker compose logs ![otp](/otp.png) +## Step 2: Download mobile and desktop app + ## Step 2: Configure apps to use your server -You can modify various Ente client apps and CLI to connect to a self hosted -custom server endpoint. +You can modify Ente mobile apps and CLI to connect to your server. ### Mobile @@ -43,7 +44,7 @@ configure the endpoint the app should be connecting to. ![Setting a custom server on the onboarding screen](custom-server.png) -### Desktop and web +### Desktop Same as the mobile app, you can tap 7 times on the onboarding screen to configure the endpoint the app should connect to. @@ -55,21 +56,6 @@ apps](web-dev-settings.png){width=400px} -This works on both the desktop app and web app (if you deploy on your own). - -To make it easier to identify when a custom server is being used, app will -thereafter show the endpoint in use (if not Ente's production server) at the -bottom of the login prompt: - -![Custom server indicator on the onboarding screen](web-custom-endpoint-indicator.png) - -Similarly, it'll be shown at other screens during the login flow. After login, -you can also see it at the bottom of the sidebar. - -Note that the custom server configured this way is cleared when you reset the -state during logout. In particular, the app also does a reset when you press the -change email button during the login flow. - ## Step 3: Configure Ente CLI > [!NOTE] diff --git a/docs/docs/self-hosting/install/quickstart.md b/docs/docs/self-hosting/install/quickstart.md index cff6e3c932..53e62898a6 100644 --- a/docs/docs/self-hosting/install/quickstart.md +++ b/docs/docs/self-hosting/install/quickstart.md @@ -3,10 +3,10 @@ title: Quickstart Script (Recommended) - Self-hosting description: Self-hosting Ente with quickstart script --- -# Quickstart +# Quickstart Script (Recommended) We provide a quickstart script which can be used for self-hosting Ente on your -machine in less than 5 minutes. +machine in less than a minute. ## Requirements @@ -25,6 +25,11 @@ The above `curl` command does the following: 1. Creates a directory `./my-ente` in working directory. 2. Starts the containers required to run Ente upon prompting. -You should be able to access the web application at [`http://localhost:3000`](http://localhost:3000) or [`http://:3000`](http://:3000) +You should be able to access the web application at [http://localhost:3000](http://localhost:3000) or [http://machine-ip:3000](http://:3000) -The data pertaining to be used by Museum is stored in `./data` folder inside `my-ente` directory, which contains extra configuration files that is to be used (billing configuration, push notification credentials, etc.) \ No newline at end of file +The data accessed by Museum is stored in `./data` folder inside `my-ente` directory. +It contains extra configuration files that is to be used (push notification credentials, etc.) + +::: tip +Check out [post-installations steps](/self-hosting/install/post-install) for further usage. +::: \ No newline at end of file diff --git a/docs/docs/self-hosting/install/requirements.md b/docs/docs/self-hosting/install/requirements.md index d0ac50b18b..822d8b1c3e 100644 --- a/docs/docs/self-hosting/install/requirements.md +++ b/docs/docs/self-hosting/install/requirements.md @@ -24,7 +24,7 @@ lightweight Go binary, since most of the intensive computational tasks are done > [!NOTE] > -> Ente requires **Docker Compose version 2.25 or higher**. +> Ente requires **Docker Compose version 2.30 or higher**. > > Furthermore, Ente uses the command `docker compose`, `docker-compose` is no > longer supported. diff --git a/docs/docs/self-hosting/install/without-docker.md b/docs/docs/self-hosting/install/without-docker.md index 826b39bb31..09f8c693fb 100644 --- a/docs/docs/self-hosting/install/without-docker.md +++ b/docs/docs/self-hosting/install/without-docker.md @@ -7,7 +7,7 @@ description: Installing and setting up Ente without Docker If you wish to run Ente from source without using Docker, follow the steps described below: -## Pre-requisites +## Requirements 1. **Go:** Install Go on your system. This is needed for building Museum (Ente's server) @@ -26,57 +26,170 @@ If you wish to run Ente from source without using Docker, follow the steps descr sudo apt install libsodium23 libsodium-dev ``` + Start the database using `systemd` automatically when the system starts. + ``` shell + sudo systemctl enable postgresql + sudo systemctl start postgresql + ``` + + Ensure the database is running using + + ``` shell + sudo systemctl status postgresql + ``` 3. **`pkg-config`:** Install `pkg-config` for dependency handling. ``` shell sudo apt install pkg-config ``` +4. **yarn, npm and Node.js:** Needed for building the web application. -Start the database using `systemd` automatically when the system starts. -``` shell -sudo systemctl enable postgresql -sudo systemctl start postgresql -``` + Install npm and Node using your package manager. -Ensure the database is running using + ``` shell + sudo apt install npm nodejs + ``` + + Install yarn by following the [official documentation](https://yarnpkg.com/getting-started/install) + +5. **Git:** Needed for cloning the repository and pulling in latest changes + +6. **Caddy:** Used for setting reverse proxy and file servers + +7. **Object Storage:** Ensure you have an object storage configured for usage, + needed for storing files. You can choose to run MinIO or Garage locally + without Docker, however, an external bucket will be reliable and suited + for long-term storage. + +## Step 1: Clone the repository + +Start by cloning Ente's repository from GitHub to your local machine. ``` shell -sudo systemctl status postgresql -``` - -### Create user - -```sh -sudo useradd postgres -``` - -## Start Museum - -Start by cloning ente to your system. - -```sh git clone https://github.com/ente-io/ente ``` -```sh -export ENTE_DB_USER=postgres -cd ente/server -go run cmd/museum/main.go -``` +## Step 2: Configure Museum (Ente's server) -You can also add the export line to your shell's RC file, to avoid exporting the environment variable every time. +1. Install all the needed dependencies for the server. + ``` shell + # Change into server directory, where the source code for Museum is + # present inside the repo + cd ente/server -## Museum as a background service + # Install the needed dependencies + go mod tidy + ``` -Please check the below links if you want to run Museum as a service, both of -them are battle tested. +2. Build the server. The server binary should be available as `./main` +relative to `server` directory -1. [How to run museum as a systemd service](https://gist.github.com/mngshm/a0edb097c91d1dc45aeed755af310323) -2. [Museum.service](https://github.com/ente-io/ente/blob/23e678889189157ecc389c258267685934b83631/server/scripts/deploy/museum.service#L4) + ``` shell + go build cmd/museum/main.go + ``` -Once you are done with setting and running Museum, all you are left to do is run -the web app and reverse_proxy it with a webserver. You can check the following -resources for Deploying your web app. +3. Create `museum.yaml` file inside `server` for configuring the needed variables. + You can copy the templated configuration file for editing with ease. -1. [Hosting the Web App](https://help.ente.io/self-hosting/guides/web-app). -2. [Running Ente Web app as a systemd Service](https://gist.github.com/mngshm/72e32bd483c2129621ed0d74412492fd) + ``` shell + cp config/example.yaml ./museum.yaml + ``` + +4. Run the server + + ``` shell + ./main + ``` + + Museum should be accessible at `http://localhost:8080` + +## Step 3: Configure Web Application + +1. Install the dependencies for web application. Enable corepack if prompted. + + ``` shell + # Change into web directory, this is where all the applications + # will be managed and built + cd web + + # Install dependencies + yarn install + ``` + +2. Configure the environment variables in your corresponding shell's configuration file + (`.bashrc`, `.zshrc`) + ``` shell + # Replace this with actual endpoint for Museum + export NEXT_PUBLIC_ENTE_ENDPOINT=http://localhost:8080 + # Replace this with actual endpoint for Albums + export NEXT_PUBLIC_ENTE_ALBUMS_ENDPOINT=http://localhost:3002 + ``` +3. Build the needed applications (Photos, Accounts, Auth, Cast) as per your needs: + ```shell + # These commands are executed inside web directory + # Build photos. Build output to be served is present at apps/photos/out + yarn build + + # Build accounts. Build output to be served is present at apps/accounts/out + yarn build:accounts + + # Build auth. Build output to be served is present at apps/auth/out + yarn build:auth + + # Build cast. Build output to be served is present at apps/cast/out + yarn build:cast + ``` + +4. Copy the output files to `/var/www/ente/apps` for easier management. + ``` shell + mkdir -p /var/www/ente/apps + + # Photos + sudo cp -r apps/photos/out /var/www/ente/apps/photos + # Accounts + sudo cp -r apps/accounts/out /var/www/ente/apps/accounts + # Auth + sudo cp -r apps/auth/out /var/www/ente/apps/auth + # Cast + sudo cp -r apps/cast/out /var/www/ente/apps/cast + ``` + +4. Set up file server using Caddy by editing `Caddyfile`, present at `/etc/caddy/Caddyfile`. + ``` groovy + # Replace the ports with domain names if you have subdomains configured and need HTTPS + :3000 { + root * /var/www/ente/apps/out/photos + file_server + try_files {path} {path}.html /index.html + } + + :3001 { + root * /var/www/ente/apps/out/accounts + file_server + try_files {path} {path}.html /index.html + } + + :3002 { + root * /var/www/ente/apps/out/photos + file_server + try_files {path} {path}.html /index.html + } + + :3003 { + root * /var/www/ente/apps/out/auth + file_server + try_files {path} {path}.html /index.html + } + + :3004 { + root * /var/www/ente/apps/out/cast + file_server + try_files {path} {path}.html /index.html + } + ``` + + The web application for Ente Photos should be accessible at http://localhost:3000, check out the [default ports](/self-hosting/install/env-var#ports) for more information. + +::: tip +Check out [post-installation steps](/self-hosting/install/post-install) for further usage. +::: \ No newline at end of file diff --git a/docs/docs/self-hosting/troubleshooting/bucket-cors.md b/docs/docs/self-hosting/troubleshooting/bucket-cors.md deleted file mode 100644 index 8bab1f7012..0000000000 --- a/docs/docs/self-hosting/troubleshooting/bucket-cors.md +++ /dev/null @@ -1,70 +0,0 @@ ---- -title: Bucket CORS -description: Troubleshooting CORS issues with S3 Buckets ---- - -# Fix potential CORS issues with your Buckets - -## For AWS S3 - -If you cannot upload a photo due to a CORS issue, you need to fix the CORS -configuration of your bucket. - -Create a `cors.json` file with the following content: - -```json -{ - "CORSRules": [ - { - "AllowedOrigins": ["*"], - "AllowedHeaders": ["*"], - "AllowedMethods": ["GET", "HEAD", "POST", "PUT", "DELETE"], - "MaxAgeSeconds": 3000, - "ExposeHeaders": ["Etag"] - } - ] -} -``` - -You may want to change the `AllowedOrigins` to a more restrictive value. - -If you are using AWS for S3, you can execute the below command to get rid of -CORS. Make sure to enter the right path for the `cors.json` file. - -```bash -aws s3api put-bucket-cors --bucket YOUR_S3_BUCKET --cors-configuration /path/to/cors.json -``` - -## For Self-hosted Minio Instance - -::: warning - -- MinIO does not support bucket CORS in the community edition which is used by - default. For more information, check - [this discussion](https://github.com/minio/minio/discussions/20841). However, - global CORS configuration is possible. -- MinIO does not take JSON CORS file as the input, instead you will have to - build a CORS.xml file or just convert the above `cors.json` to XML. - -::: - -A minor requirement here is the tool `mc` for managing buckets via command line -interface. Checkout the `mc set alias` document to configure alias for your -instance and bucket. After this you will be prompted for your AccessKey and -Secret, which is your username and password. - -```sh -mc cors set // api cors_allow_origin="*" -``` - -You can create also `.csv` file and dump the list of origins you would like to -allow and replace the `*` with `path` to the CSV file. - -Now, uploads should be working fine. diff --git a/docs/docs/self-hosting/troubleshooting/keyring.md b/docs/docs/self-hosting/troubleshooting/keyring.md index 56a5807fa5..a521a13c27 100644 --- a/docs/docs/self-hosting/troubleshooting/keyring.md +++ b/docs/docs/self-hosting/troubleshooting/keyring.md @@ -13,9 +13,8 @@ Follow the below steps to run Ente CLI and also avoid keyrings errors. Run: -```sh +``` shell # export the secrets path - export ENTE_CLI_SECRETS_PATH=./ ./ente-cli @@ -33,7 +32,7 @@ Then one of the following: And you are good to go. -## Ref +## References - [Ente CLI Secrets Path](https://www.reddit.com/r/selfhosted/comments/1gc09il/comment/lu2hox2/?utm_source=share&utm_medium=web3x&utm_name=web3xcss&utm_term=1&utm_content=share_button) - [Keyrings](https://man7.org/linux/man-pages/man7/keyrings.7.html) diff --git a/docs/docs/self-hosting/troubleshooting/uploads.md b/docs/docs/self-hosting/troubleshooting/uploads.md index 6cad97f201..fd22da1271 100644 --- a/docs/docs/self-hosting/troubleshooting/uploads.md +++ b/docs/docs/self-hosting/troubleshooting/uploads.md @@ -17,16 +17,6 @@ It is also suggested that the user setups bucket CORS or global CORS on MinIO or any external S3 service provider they are connecting to. To setup bucket CORS, please [read this](/self-hosting/troubleshooting/bucket-cors). -## What is S3 and how is it incorporated in Ente ? - -S3 is an cloud storage protocol made by Amazon (specifically AWS). S3 is -designed to store files and data as objects inside buckets and it is mostly used -for online backups and storing different types of files. - -Ente's Docker setup is shipped with [MinIO](https://min.io/) as its default S3 -provider. MinIO supports the Amazon S3 protocol and leverages your disk storage -to dump all the uploaded files as encrypted object blobs. - ## 403 Forbidden If museum is able to make a network connection to your S3 bucket but uploads are diff --git a/server/.gitignore b/server/.gitignore index 0a1c4aa635..9a150a8838 100644 --- a/server/.gitignore +++ b/server/.gitignore @@ -11,4 +11,5 @@ data/ my-ente/ __debug_bin* config/.env -config/museum.yaml \ No newline at end of file +config/museum.yaml +main \ No newline at end of file diff --git a/server/config/example.yaml b/server/config/example.yaml index 5e0f94ece1..61524df6f5 100644 --- a/server/config/example.yaml +++ b/server/config/example.yaml @@ -1,5 +1,13 @@ # Copy this file to museum.yaml in the same directory in which this file is present +# This section is meant for configuration of database. +# Museum uses these values and credentials for connecting +# to the database. +# Set a strong password and if using PostgreSQL Docker container +# provided in Docker Compose file, ensure db.password is same +# as POSTGRES_PASSWORD +# Similarly ensure db.user and db.name are same as POSTGRES_USER and +# POSTGRES_DB respectively db: host: postgres port: 5432 @@ -7,15 +15,27 @@ db: user: pguser password: +# This section is for configuring storage buckets. Omit this section if +# you only intend to use Ente Auth s3: + # Change this to false if enabling SSL are_local_buckets: true + # Only path-style URL works if disabling are_local_buckets with MinIO + use_path_style_urls: true b2-eu-cen: + # You can uncomment the below 2 lines of configuration to + # configure SSL and URL style per bucket. + # If not configured, top-level configuration is used. + # are_local_buckets: true + # use_path_style_urls: true key: secret: endpoint: localhost:3200 region: eu-central-2 bucket: b2-eu-cen wasabi-eu-central-2-v3: + # are_local_buckets: true + # use_path_style_urls: true key: secret: endpoint: localhost:3200 @@ -23,8 +43,34 @@ s3: bucket: wasabi-eu-central-2-v3 compliance: false scw-eu-fr-v3: + # are_local_buckets: true + # use_path_style_urls: true key: secret: endpoint: localhost:3200 region: eu-central-2 - bucket: scw-eu-fr-v3 \ No newline at end of file + bucket: scw-eu-fr-v3 + +# Key used for encrypting customer emails before storing them in DB +# +# To make it easy to get started, some randomly generated (but fixed) values are +# provided here. But if you're really going to be using museum, please generate +# new keys. You can use `go run tools/gen-random-keys/main.go` for that. +# +# Replace values in key and JWT for security +key: + encryption: yvmG/RnzKrbCb9L3mgsmoxXr9H7i2Z4qlbT0mL3ln4w= + hash: KXYiG07wC7GIgvCSdg+WmyWdXDAn6XKYJtp/wkEU7x573+byBRAYtpTP0wwvi8i/4l37uicX1dVTUzwH3sLZyw== +# JWT secrets +jwt: + secret: i2DecQmfGreG6q1vBj5tCokhlN41gcfS2cjOs9Po-u8= + +# Specify the base endpoints for various web apps +apps: + # If you're running a self hosted instance and wish to serve public links, + # set this to the URL where your albums web app is running. + public-albums: http://localhost:3002 + cast: http://localhost:3004 + # Set this to the URL where your accounts web app is running, primarily used for + # passkey based 2FA. + accounts: http://localhost:3001 diff --git a/server/quickstart.sh b/server/quickstart.sh index 74535d437e..a656437917 100755 --- a/server/quickstart.sh +++ b/server/quickstart.sh @@ -159,13 +159,6 @@ printf " \033[1;32mN\033[0m Created \033[1mcompose.yaml\033[0m\n" sleep 1 cat <museum.yaml -key: - encryption: $museum_key - hash: $museum_hash - -jwt: - secret: $museum_jwt_secret - db: host: postgres port: 5432 @@ -194,6 +187,23 @@ s3: endpoint: localhost:3200 region: eu-central-2 bucket: scw-eu-fr-v3 + +# Specify the base endpoints for various web apps +apps: + # If you're running a self hosted instance and wish to serve public links, + # set this to the URL where your albums web app is running. + public-albums: http://localhost:3002 + cast: http://localhost:3004 + # Set this to the URL where your accounts web app is running, primarily used for + # passkey based 2FA. + accounts: http://localhost:3001 + +key: + encryption: $museum_key + hash: $museum_hash + +jwt: + secret: $museum_jwt_secret EOF printf " \033[1;32mT\033[0m Created \033[1mmuseum.yaml\033[0m\n"