From 99d9a55e8d6a72028e34213fe8189dd2226384cd Mon Sep 17 00:00:00 2001 From: Mikael Finstad Date: Mon, 21 Nov 2022 14:02:41 +0800 Subject: [PATCH 1/9] add docs for COMPANION_PREAUTH_SECRET --- .env.example | 1 + bin/companion.sh | 1 + packages/@uppy/companion/KUBERNETES.md | 1 + packages/@uppy/companion/env_example | 1 + packages/@uppy/companion/test/mockserver.js | 1 + website/src/docs/companion.md | 5 +++++ 6 files changed, 10 insertions(+) diff --git a/.env.example b/.env.example index 61023c445c..1d76d34b33 100644 --- a/.env.example +++ b/.env.example @@ -10,6 +10,7 @@ COMPANION_PROTOCOL=http COMPANION_PORT=3020 COMPANION_CLIENT_ORIGINS= COMPANION_SECRET=development +COMPANION_PREAUTH_SECRET=development2 # NOTE: Only enable this in development. Enabling it in production is a security risk COMPANION_ALLOW_LOCAL_URLS=true diff --git a/bin/companion.sh b/bin/companion.sh index df114b8b35..4b31af6684 100755 --- a/bin/companion.sh +++ b/bin/companion.sh @@ -11,6 +11,7 @@ else COMPANION_PORT=3020 \ COMPANION_CLIENT_ORIGINS="" \ COMPANION_SECRET="development" \ + COMPANION_PREAUTH_SECRET="development2" \ COMPANION_ALLOW_LOCAL_URLS="true" \ nodemon --watch packages/@uppy/companion/src --exec node ./packages/@uppy/companion/src/standalone/start-server.js fi diff --git a/packages/@uppy/companion/KUBERNETES.md b/packages/@uppy/companion/KUBERNETES.md index 3cb6405b97..0198135e85 100644 --- a/packages/@uppy/companion/KUBERNETES.md +++ b/packages/@uppy/companion/KUBERNETES.md @@ -28,6 +28,7 @@ data: COMPANION_STREAMING_UPLOAD: true COMPANION_REDIS_URL: redis://:superSecretPassword@uppy-redis.uppy.svc.cluster.local:6379 COMPANION_SECRET: "shh!Issa Secret!" + COMPANION_PREAUTH_SECRET: "another secret" COMPANION_DROPBOX_KEY: "YOUR DROPBOX KEY" COMPANION_DROPBOX_SECRET: "YOUR DROPBOX SECRET" COMPANION_BOX_KEY: "YOUR BOX KEY" diff --git a/packages/@uppy/companion/env_example b/packages/@uppy/companion/env_example index 1bddc277c4..fe4b1aac30 100644 --- a/packages/@uppy/companion/env_example +++ b/packages/@uppy/companion/env_example @@ -9,6 +9,7 @@ COMPANION_STREAMING_UPLOAD=true COMPANION_PROTOCOL=https COMPANION_DATADIR=/mnt/uppy-server-data COMPANION_SECRET= +COMPANION_PREAUTH_SECRET= COMPANION_SECRET_FILE= COMPANION_DROPBOX_KEY="dropbox_key" diff --git a/packages/@uppy/companion/test/mockserver.js b/packages/@uppy/companion/test/mockserver.js index b464330b73..15dfd8f8d4 100644 --- a/packages/@uppy/companion/test/mockserver.js +++ b/packages/@uppy/companion/test/mockserver.js @@ -17,6 +17,7 @@ const defaultEnv = { COMPANION_PROTOCOL: 'http', COMPANION_DATADIR: './test/output', COMPANION_SECRET: 'secret', + COMPANION_PREAUTH_SECRET: 'different secret', COMPANION_DROPBOX_KEY: 'dropbox_key', COMPANION_DROPBOX_SECRET: 'dropbox_secret', diff --git a/website/src/docs/companion.md b/website/src/docs/companion.md index 9d6624b68c..48fde87d14 100644 --- a/website/src/docs/companion.md +++ b/website/src/docs/companion.md @@ -295,6 +295,11 @@ export COMPANION_PERIODIC_PING_URLS="https://example.com/ping1,https://example.c export COMPANION_PERIODIC_PING_INTERVAL=60000 # corresponds to the periodicPingStaticPayload option (JSON string) export COMPANION_PERIODIC_PING_STATIC_JSON_PAYLOAD="{\"static\":\"data\"}" + +# If you need to use `companionKeysParams` (custom OAuth credentials at request time), +# set this variable to a strong randomly generated secret. +# See also https://github.com/transloadit/uppy/pull/2622 +COMPANION_PREAUTH_SECRET="preauth secret" ``` See [`.env.example`](https://github.com/transloadit/uppy/blob/main/.env.example) for an example environment configuration file. From 2dbbe79424ecccd6fbbbf2cb2e3e6904316164bf Mon Sep 17 00:00:00 2001 From: Mikael Finstad Date: Mon, 21 Nov 2022 14:23:31 +0800 Subject: [PATCH 2/9] document Companion multi-instance feature #3538 --- website/src/docs/companion.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/website/src/docs/companion.md b/website/src/docs/companion.md index 48fde87d14..146f1981f7 100644 --- a/website/src/docs/companion.md +++ b/website/src/docs/companion.md @@ -486,6 +486,25 @@ app.use(companion.app({ We have [a detailed guide on running Companion in Kubernetes](https://github.com/transloadit/uppy/blob/main/packages/%40uppy/companion/KUBERNETES.md) for you, that’s how we run our example server at . +### Running many instances + +Two ways of running many concurrent Companion instances. + +#### Separate domain names + +One option is to run many instances with each instance having its own (sub)domain name. With this setup, the Companion client in Uppy will direct requests to companion in a way so that all requests of a particular upload will always be sent to the same instance. You would then also typically configure a single instance (one domain name) to handle all OAuth authentication requests, so that you only need to specify a single OAuth callback URL. See also `oauthDomain` and `validHosts` + +#### Behind a load balancer + +The other option is to set up a load balancer in front of many Companion instances. Then Uppys companion client will only see a single domain name and send all requests to the load balancer, which will then distribute them evenly between Companion instances. The companion instances will then coordinate their messages and events over Redis so that any instance can serve the client’s requests. Note that sticky sessions are **not** needed with this setup. Here are some requirements for this setup: + +* The instances need to be connected to the same Redis server. +* You need to set `COMPANION_SECRET` to the same value on both servers. +* if you use the `companionKeysParams` feature (Transloadit), you also need `COMPANION_PREAUTH_SECRET` to be the same on each instance. +* All other configuration needs to be the same, except if you’re running many instances on the same machine, then `COMPANION_PORT` should be different for each instance. + +For more information about this setup [see this issue](https://github.com/transloadit/uppy/issues/3538). + ### Adding custom providers As of now, Companion supports the [providers listed here](https://uppy.io/docs/companion/#Supported-providers) out of the box, but you may also choose to add your own custom providers. You can do this by passing the `customProviders` option when calling the Uppy `app` method. The custom provider is expected to support Oauth 1 or 2 for authentication/authorization. From 96fec3cab4c3ced02c656756389d4db657f2574a Mon Sep 17 00:00:00 2001 From: Mikael Finstad Date: Mon, 21 Nov 2022 21:08:53 +0800 Subject: [PATCH 3/9] Apply suggestions from code review Co-authored-by: Kevin van Zonneveld --- website/src/docs/companion.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/website/src/docs/companion.md b/website/src/docs/companion.md index 146f1981f7..3ea3e0b439 100644 --- a/website/src/docs/companion.md +++ b/website/src/docs/companion.md @@ -490,13 +490,13 @@ We have [a detailed guide on running Companion in Kubernetes](https://github.com Two ways of running many concurrent Companion instances. -#### Separate domain names +#### Separate endpoints -One option is to run many instances with each instance having its own (sub)domain name. With this setup, the Companion client in Uppy will direct requests to companion in a way so that all requests of a particular upload will always be sent to the same instance. You would then also typically configure a single instance (one domain name) to handle all OAuth authentication requests, so that you only need to specify a single OAuth callback URL. See also `oauthDomain` and `validHosts` +One option is to run many instances with each instance having its own unique endpoint. This could be on separate ports, (sub)domain names, or IPs. With this setup, the Companion client in Uppy will direct requests to companion in a way so that all requests of a particular upload will always be sent to the same endpoint, and hence process. You would then also typically configure a single instance (one endpoint) to handle all OAuth authentication requests, so that you only need to specify a single OAuth callback URL. See also `oauthDomain` and `validHosts` #### Behind a load balancer -The other option is to set up a load balancer in front of many Companion instances. Then Uppys companion client will only see a single domain name and send all requests to the load balancer, which will then distribute them evenly between Companion instances. The companion instances will then coordinate their messages and events over Redis so that any instance can serve the client’s requests. Note that sticky sessions are **not** needed with this setup. Here are some requirements for this setup: +The other option is to set up a load balancer in front of many Companion instances. Then Uppy’s companion client will only see a single endpoint and send all requests to the associated load balancer, which will then distribute them between Companion instances. The companion instances will then coordinate their messages and events over Redis so that any instance can serve the client’s requests. Note that sticky sessions are **not** needed with this setup. Here are some requirements for this setup: * The instances need to be connected to the same Redis server. * You need to set `COMPANION_SECRET` to the same value on both servers. From 4f3a0446a9fe1edf547b607b78b8b0b3270bba1a Mon Sep 17 00:00:00 2001 From: Mikael Finstad Date: Tue, 22 Nov 2022 16:23:05 +0800 Subject: [PATCH 4/9] Update website/src/docs/companion.md --- website/src/docs/companion.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/website/src/docs/companion.md b/website/src/docs/companion.md index 3ea3e0b439..02f5a6430d 100644 --- a/website/src/docs/companion.md +++ b/website/src/docs/companion.md @@ -503,8 +503,6 @@ The other option is to set up a load balancer in front of many Companion instanc * if you use the `companionKeysParams` feature (Transloadit), you also need `COMPANION_PREAUTH_SECRET` to be the same on each instance. * All other configuration needs to be the same, except if you’re running many instances on the same machine, then `COMPANION_PORT` should be different for each instance. -For more information about this setup [see this issue](https://github.com/transloadit/uppy/issues/3538). - ### Adding custom providers As of now, Companion supports the [providers listed here](https://uppy.io/docs/companion/#Supported-providers) out of the box, but you may also choose to add your own custom providers. You can do this by passing the `customProviders` option when calling the Uppy `app` method. The custom provider is expected to support Oauth 1 or 2 for authentication/authorization. From 65143fd3a4a22919bddf9dd848543dafa9ce8320 Mon Sep 17 00:00:00 2001 From: Mikael Finstad Date: Tue, 22 Nov 2022 17:01:56 +0800 Subject: [PATCH 5/9] explain better how i-am works --- website/src/docs/companion.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/website/src/docs/companion.md b/website/src/docs/companion.md index 02f5a6430d..3e001ad92a 100644 --- a/website/src/docs/companion.md +++ b/website/src/docs/companion.md @@ -488,11 +488,15 @@ We have [a detailed guide on running Companion in Kubernetes](https://github.com ### Running many instances -Two ways of running many concurrent Companion instances. +Two ways of running many concurrent Companion instances. It is always recommended to run at least two instances in production, so that if the Node.js event loop gets blocked by one or more requests, it doesn’t also block all other requests (due to Node.js single threaded nature.) #### Separate endpoints -One option is to run many instances with each instance having its own unique endpoint. This could be on separate ports, (sub)domain names, or IPs. With this setup, the Companion client in Uppy will direct requests to companion in a way so that all requests of a particular upload will always be sent to the same endpoint, and hence process. You would then also typically configure a single instance (one endpoint) to handle all OAuth authentication requests, so that you only need to specify a single OAuth callback URL. See also `oauthDomain` and `validHosts` +One option is to run many instances with each instance having its own unique endpoint. This could be on separate ports, (sub)domain names, or IPs. With this setup, you can either +1. Implement your own logic that will direct each upload to a specific Companion endpoint by setting the `companionUrl` option +2. Setting the Companion option `COMPANION_SELF_ENDPOINT`. This option will cause Companion to respond with a special `i-am` HTTP header containing the value from `COMPANION_SELF_ENDPOINT`. When Uppy’s Companion client sees this header, it will pin all requests for the upload to this endpoint. + +In either case, you would then also typically configure a single Companion instance (one endpoint) to handle all OAuth authentication requests, so that you only need to specify a single OAuth callback URL. See also `oauthDomain` and `validHosts`. #### Behind a load balancer From eb591896150e14d7e50282a6e4399e43073e0252 Mon Sep 17 00:00:00 2001 From: Mikael Finstad Date: Tue, 22 Nov 2022 17:07:49 +0800 Subject: [PATCH 6/9] Add experience from Tranloadit --- website/src/docs/companion.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/website/src/docs/companion.md b/website/src/docs/companion.md index 3e001ad92a..198d1dc82e 100644 --- a/website/src/docs/companion.md +++ b/website/src/docs/companion.md @@ -488,7 +488,11 @@ We have [a detailed guide on running Companion in Kubernetes](https://github.com ### Running many instances -Two ways of running many concurrent Companion instances. It is always recommended to run at least two instances in production, so that if the Node.js event loop gets blocked by one or more requests, it doesn’t also block all other requests (due to Node.js single threaded nature.) +Two ways of running many concurrent Companion instances. + +In our experience Companion will saturate network interface cards before other resources on commodity virtual servers (`c5d.2xlarge` for instance). We recommend running at least two instances in production, so that if the Node.js event loop gets blocked by one or more requests (due to a bug or spike in traffic), it doesn’t also block or slow down all other requests as well (due to Node.js single threaded nature.) As an example for scale, one OSS enterprise customer of Transloadit that self-hosts Companion to power an education service that is deployed by virtually all universities globally, deploys 7 Companion instances, their previous solution ran on 35 instances I believe. + +But as always it depends and your mileage may vary, so we recommend to add observability. You can let Prometheus crawl the /metrics endpoint and graph that with Grafana for instance. #### Separate endpoints From efa4c64de3789bba0b864d444859aabd81cdd519 Mon Sep 17 00:00:00 2001 From: Mikael Finstad Date: Tue, 22 Nov 2022 17:11:28 +0800 Subject: [PATCH 7/9] Update companion.md --- website/src/docs/companion.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/src/docs/companion.md b/website/src/docs/companion.md index 198d1dc82e..fc12a8c6f9 100644 --- a/website/src/docs/companion.md +++ b/website/src/docs/companion.md @@ -490,7 +490,7 @@ We have [a detailed guide on running Companion in Kubernetes](https://github.com Two ways of running many concurrent Companion instances. -In our experience Companion will saturate network interface cards before other resources on commodity virtual servers (`c5d.2xlarge` for instance). We recommend running at least two instances in production, so that if the Node.js event loop gets blocked by one or more requests (due to a bug or spike in traffic), it doesn’t also block or slow down all other requests as well (due to Node.js single threaded nature.) As an example for scale, one OSS enterprise customer of Transloadit that self-hosts Companion to power an education service that is deployed by virtually all universities globally, deploys 7 Companion instances, their previous solution ran on 35 instances I believe. +In our experience Companion will saturate network interface cards before other resources on commodity virtual servers (`c5d.2xlarge` for instance). We recommend running at least two instances in production, so that if the Node.js event loop gets blocked by one or more requests (due to a bug or spike in traffic), it doesn’t also block or slow down all other requests as well (due to Node.js single threaded nature.) As an example for scale, one OSS enterprise customer of Transloadit that self-hosts Companion to power an education service that is deployed by virtually all universities globally, deploys 7 Companion instances, their earlier solution ran on 35 instances I believe. But as always it depends and your mileage may vary, so we recommend to add observability. You can let Prometheus crawl the /metrics endpoint and graph that with Grafana for instance. From 3bb9b1897c1c96bc8e4ab01a22c3f735a027983b Mon Sep 17 00:00:00 2001 From: Mikael Finstad Date: Wed, 23 Nov 2022 09:54:01 +0800 Subject: [PATCH 8/9] Apply suggestions from code review Co-authored-by: Merlijn Vos --- website/src/docs/companion.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/website/src/docs/companion.md b/website/src/docs/companion.md index fc12a8c6f9..2d9bf4aeee 100644 --- a/website/src/docs/companion.md +++ b/website/src/docs/companion.md @@ -488,23 +488,24 @@ We have [a detailed guide on running Companion in Kubernetes](https://github.com ### Running many instances -Two ways of running many concurrent Companion instances. -In our experience Companion will saturate network interface cards before other resources on commodity virtual servers (`c5d.2xlarge` for instance). We recommend running at least two instances in production, so that if the Node.js event loop gets blocked by one or more requests (due to a bug or spike in traffic), it doesn’t also block or slow down all other requests as well (due to Node.js single threaded nature.) As an example for scale, one OSS enterprise customer of Transloadit that self-hosts Companion to power an education service that is deployed by virtually all universities globally, deploys 7 Companion instances, their earlier solution ran on 35 instances I believe. +In our experience Companion will saturate network interface cards before other resources on commodity virtual servers (`c5d.2xlarge` for instance). We recommend running at least two instances in production, so that if the Node.js event loop gets blocked by one or more requests (due to a bug or spike in traffic), it doesn’t also block or slow down all other requests as well (as Node.js is single threaded). -But as always it depends and your mileage may vary, so we recommend to add observability. You can let Prometheus crawl the /metrics endpoint and graph that with Grafana for instance. +As an example for scale, one enterprise customer of Transloadit, who self-hosts Companion to power an education service that is used by many universities globally, deploys 7 Companion instances. Their earlier solution ran on 35 instances I believe. + +Your mileage may vary, so we recommend to add observability. You can let Prometheus crawl the `/metrics` endpoint and graph that with Grafana for instance. #### Separate endpoints One option is to run many instances with each instance having its own unique endpoint. This could be on separate ports, (sub)domain names, or IPs. With this setup, you can either 1. Implement your own logic that will direct each upload to a specific Companion endpoint by setting the `companionUrl` option -2. Setting the Companion option `COMPANION_SELF_ENDPOINT`. This option will cause Companion to respond with a special `i-am` HTTP header containing the value from `COMPANION_SELF_ENDPOINT`. When Uppy’s Companion client sees this header, it will pin all requests for the upload to this endpoint. +2. Setting the Companion option `COMPANION_SELF_ENDPOINT`. This option will cause Companion to respond with a `i-am` HTTP header containing the value from `COMPANION_SELF_ENDPOINT`. When Uppy’s sees this header, it will pin all requests for the upload to this endpoint. In either case, you would then also typically configure a single Companion instance (one endpoint) to handle all OAuth authentication requests, so that you only need to specify a single OAuth callback URL. See also `oauthDomain` and `validHosts`. -#### Behind a load balancer +#### Using a load balancer -The other option is to set up a load balancer in front of many Companion instances. Then Uppy’s companion client will only see a single endpoint and send all requests to the associated load balancer, which will then distribute them between Companion instances. The companion instances will then coordinate their messages and events over Redis so that any instance can serve the client’s requests. Note that sticky sessions are **not** needed with this setup. Here are some requirements for this setup: +The other option is to set up a load balancer in front of many Companion instances. Then Uppy will only see a single endpoint and send all requests to the associated load balancer, which will then distribute them between Companion instances. The companion instances coordinate their messages and events over Redis so that any instance can serve the client’s requests. Note that sticky sessions are **not** needed with this setup. Here are the requirements for this setup: * The instances need to be connected to the same Redis server. * You need to set `COMPANION_SECRET` to the same value on both servers. From 48843f5309029c2c3473be4d970eabece80a014a Mon Sep 17 00:00:00 2001 From: Mikael Finstad Date: Thu, 24 Nov 2022 09:57:01 +0800 Subject: [PATCH 9/9] Apply suggestions from code review Co-authored-by: Merlijn Vos --- website/src/docs/companion.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/website/src/docs/companion.md b/website/src/docs/companion.md index 2d9bf4aeee..078f5c52cb 100644 --- a/website/src/docs/companion.md +++ b/website/src/docs/companion.md @@ -489,13 +489,13 @@ We have [a detailed guide on running Companion in Kubernetes](https://github.com ### Running many instances -In our experience Companion will saturate network interface cards before other resources on commodity virtual servers (`c5d.2xlarge` for instance). We recommend running at least two instances in production, so that if the Node.js event loop gets blocked by one or more requests (due to a bug or spike in traffic), it doesn’t also block or slow down all other requests as well (as Node.js is single threaded). +We recommend running at least two instances in production, so that if the Node.js event loop gets blocked by one or more requests (due to a bug or spike in traffic), it doesn’t also block or slow down all other requests as well (as Node.js is single threaded). -As an example for scale, one enterprise customer of Transloadit, who self-hosts Companion to power an education service that is used by many universities globally, deploys 7 Companion instances. Their earlier solution ran on 35 instances I believe. +As an example for scale, one enterprise customer of Transloadit, who self-hosts Companion to power an education service that is used by many universities globally, deploys 7 Companion instances. Their earlier solution ran on 35 instances. In our general experience Companion will saturate network interface cards before other resources on commodity virtual servers (`c5d.2xlarge` for instance). Your mileage may vary, so we recommend to add observability. You can let Prometheus crawl the `/metrics` endpoint and graph that with Grafana for instance. -#### Separate endpoints +#### Using unique endpoints One option is to run many instances with each instance having its own unique endpoint. This could be on separate ports, (sub)domain names, or IPs. With this setup, you can either 1. Implement your own logic that will direct each upload to a specific Companion endpoint by setting the `companionUrl` option