diff --git a/.env.example b/.env.example index 61023c445c..1d76d34b33 100644 --- a/.env.example +++ b/.env.example @@ -10,6 +10,7 @@ COMPANION_PROTOCOL=http COMPANION_PORT=3020 COMPANION_CLIENT_ORIGINS= COMPANION_SECRET=development +COMPANION_PREAUTH_SECRET=development2 # NOTE: Only enable this in development. Enabling it in production is a security risk COMPANION_ALLOW_LOCAL_URLS=true diff --git a/bin/companion.sh b/bin/companion.sh index df114b8b35..4b31af6684 100755 --- a/bin/companion.sh +++ b/bin/companion.sh @@ -11,6 +11,7 @@ else COMPANION_PORT=3020 \ COMPANION_CLIENT_ORIGINS="" \ COMPANION_SECRET="development" \ + COMPANION_PREAUTH_SECRET="development2" \ COMPANION_ALLOW_LOCAL_URLS="true" \ nodemon --watch packages/@uppy/companion/src --exec node ./packages/@uppy/companion/src/standalone/start-server.js fi diff --git a/packages/@uppy/companion/KUBERNETES.md b/packages/@uppy/companion/KUBERNETES.md index 3cb6405b97..0198135e85 100644 --- a/packages/@uppy/companion/KUBERNETES.md +++ b/packages/@uppy/companion/KUBERNETES.md @@ -28,6 +28,7 @@ data: COMPANION_STREAMING_UPLOAD: true COMPANION_REDIS_URL: redis://:superSecretPassword@uppy-redis.uppy.svc.cluster.local:6379 COMPANION_SECRET: "shh!Issa Secret!" + COMPANION_PREAUTH_SECRET: "another secret" COMPANION_DROPBOX_KEY: "YOUR DROPBOX KEY" COMPANION_DROPBOX_SECRET: "YOUR DROPBOX SECRET" COMPANION_BOX_KEY: "YOUR BOX KEY" diff --git a/packages/@uppy/companion/env_example b/packages/@uppy/companion/env_example index 1bddc277c4..fe4b1aac30 100644 --- a/packages/@uppy/companion/env_example +++ b/packages/@uppy/companion/env_example @@ -9,6 +9,7 @@ COMPANION_STREAMING_UPLOAD=true COMPANION_PROTOCOL=https COMPANION_DATADIR=/mnt/uppy-server-data COMPANION_SECRET= +COMPANION_PREAUTH_SECRET= COMPANION_SECRET_FILE= COMPANION_DROPBOX_KEY="dropbox_key" diff --git a/packages/@uppy/companion/test/mockserver.js b/packages/@uppy/companion/test/mockserver.js index b464330b73..15dfd8f8d4 100644 --- a/packages/@uppy/companion/test/mockserver.js +++ b/packages/@uppy/companion/test/mockserver.js @@ -17,6 +17,7 @@ const defaultEnv = { COMPANION_PROTOCOL: 'http', COMPANION_DATADIR: './test/output', COMPANION_SECRET: 'secret', + COMPANION_PREAUTH_SECRET: 'different secret', COMPANION_DROPBOX_KEY: 'dropbox_key', COMPANION_DROPBOX_SECRET: 'dropbox_secret', diff --git a/website/src/docs/companion.md b/website/src/docs/companion.md index 9d6624b68c..078f5c52cb 100644 --- a/website/src/docs/companion.md +++ b/website/src/docs/companion.md @@ -295,6 +295,11 @@ export COMPANION_PERIODIC_PING_URLS="https://example.com/ping1,https://example.c export COMPANION_PERIODIC_PING_INTERVAL=60000 # corresponds to the periodicPingStaticPayload option (JSON string) export COMPANION_PERIODIC_PING_STATIC_JSON_PAYLOAD="{\"static\":\"data\"}" + +# If you need to use `companionKeysParams` (custom OAuth credentials at request time), +# set this variable to a strong randomly generated secret. +# See also https://github.com/transloadit/uppy/pull/2622 +COMPANION_PREAUTH_SECRET="preauth secret" ``` See [`.env.example`](https://github.com/transloadit/uppy/blob/main/.env.example) for an example environment configuration file. @@ -481,6 +486,32 @@ app.use(companion.app({ We have [a detailed guide on running Companion in Kubernetes](https://github.com/transloadit/uppy/blob/main/packages/%40uppy/companion/KUBERNETES.md) for you, that’s how we run our example server at . +### Running many instances + + +We recommend running at least two instances in production, so that if the Node.js event loop gets blocked by one or more requests (due to a bug or spike in traffic), it doesn’t also block or slow down all other requests as well (as Node.js is single threaded). + +As an example for scale, one enterprise customer of Transloadit, who self-hosts Companion to power an education service that is used by many universities globally, deploys 7 Companion instances. Their earlier solution ran on 35 instances. In our general experience Companion will saturate network interface cards before other resources on commodity virtual servers (`c5d.2xlarge` for instance). + +Your mileage may vary, so we recommend to add observability. You can let Prometheus crawl the `/metrics` endpoint and graph that with Grafana for instance. + +#### Using unique endpoints + +One option is to run many instances with each instance having its own unique endpoint. This could be on separate ports, (sub)domain names, or IPs. With this setup, you can either +1. Implement your own logic that will direct each upload to a specific Companion endpoint by setting the `companionUrl` option +2. Setting the Companion option `COMPANION_SELF_ENDPOINT`. This option will cause Companion to respond with a `i-am` HTTP header containing the value from `COMPANION_SELF_ENDPOINT`. When Uppy’s sees this header, it will pin all requests for the upload to this endpoint. + +In either case, you would then also typically configure a single Companion instance (one endpoint) to handle all OAuth authentication requests, so that you only need to specify a single OAuth callback URL. See also `oauthDomain` and `validHosts`. + +#### Using a load balancer + +The other option is to set up a load balancer in front of many Companion instances. Then Uppy will only see a single endpoint and send all requests to the associated load balancer, which will then distribute them between Companion instances. The companion instances coordinate their messages and events over Redis so that any instance can serve the client’s requests. Note that sticky sessions are **not** needed with this setup. Here are the requirements for this setup: + +* The instances need to be connected to the same Redis server. +* You need to set `COMPANION_SECRET` to the same value on both servers. +* if you use the `companionKeysParams` feature (Transloadit), you also need `COMPANION_PREAUTH_SECRET` to be the same on each instance. +* All other configuration needs to be the same, except if you’re running many instances on the same machine, then `COMPANION_PORT` should be different for each instance. + ### Adding custom providers As of now, Companion supports the [providers listed here](https://uppy.io/docs/companion/#Supported-providers) out of the box, but you may also choose to add your own custom providers. You can do this by passing the `customProviders` option when calling the Uppy `app` method. The custom provider is expected to support Oauth 1 or 2 for authentication/authorization.