diff --git a/.github/workflows/local_tests.yaml b/.github/workflows/local_tests.yaml index 6421bcaf3..bf09457e0 100644 --- a/.github/workflows/local_tests.yaml +++ b/.github/workflows/local_tests.yaml @@ -5,6 +5,9 @@ on: branches: [ main ] pull_request: +env: + API_SERVER_URL: https://api.run.house + jobs: # TODO: THESE ARE ONLY SEPARATE JOBS BECAUSE THERE ARE # DEVELOPMENT INCONSISTENCIES WHEN RUNNING ALL THE LOCAL @@ -38,6 +41,7 @@ jobs: with: username: ${{ secrets.CI_ACCOUNT_USERNAME }} token: ${{ secrets.CI_ACCOUNT_TOKEN }} + api_server_url: ${{ env.API_SERVER_URL }} - name: pytest -v --level local tests/test_servers/ env: @@ -75,6 +79,7 @@ jobs: with: username: ${{ secrets.CI_ACCOUNT_USERNAME }} token: ${{ secrets.CI_ACCOUNT_TOKEN }} + api_server_url: ${{ env.API_SERVER_URL }} - name: pytest -v --level local -k "not servertest and not secrettest and not moduletest and not functiontest and not envtest" env: @@ -112,6 +117,7 @@ jobs: with: username: ${{ secrets.CI_ACCOUNT_USERNAME }} token: ${{ secrets.CI_ACCOUNT_TOKEN }} + api_server_url: ${{ env.API_SERVER_URL }} - name: pytest -v --level local -k "secrettest" env: @@ -134,6 +140,7 @@ jobs: with: username: ${{ secrets.CI_ACCOUNT_USERNAME }} token: ${{ secrets.CI_ACCOUNT_TOKEN }} + api_server_url: ${{ env.API_SERVER_URL }} - name: pytest -v --level local -k "moduletest" env: @@ -156,6 +163,7 @@ jobs: with: username: ${{ secrets.CI_ACCOUNT_USERNAME }} token: ${{ secrets.CI_ACCOUNT_TOKEN }} + api_server_url: ${{ env.API_SERVER_URL }} - name: pytest -v --level local -k "functiontest" env: @@ -178,6 +186,7 @@ jobs: with: username: ${{ secrets.CI_ACCOUNT_USERNAME }} token: ${{ secrets.CI_ACCOUNT_TOKEN }} + api_server_url: ${{ env.API_SERVER_URL }} - name: pytest -v --level local -k "envtest" env: diff --git a/.github/workflows/local_tests_den_dev.yaml b/.github/workflows/local_tests_den_dev.yaml new file mode 100644 index 000000000..9cf48eb9d --- /dev/null +++ b/.github/workflows/local_tests_den_dev.yaml @@ -0,0 +1,195 @@ +name: Tests with level "local" on Den Dev + +on: + workflow_dispatch: + +env: + API_SERVER_URL: https://api-dev.run.house + +jobs: + # TODO: THESE ARE ONLY SEPARATE JOBS BECAUSE THERE ARE + # DEVELOPMENT INCONSISTENCIES WHEN RUNNING ALL THE LOCAL + # TESTS TOGETHER. + # server-tests-logged-out-level-local: + # runs-on: ubuntu-latest + # steps: + # - name: Check out repository code + # uses: actions/checkout@v3 + + # - name: Setup Runhouse + # uses: ./.github/workflows/setup_runhouse + + # - name: pytest -v --level local tests/test_servers/ + # env: + # TEST_TOKEN: ${{ secrets.TEST_TOKEN }} + # TEST_USERNAME: ${{ secrets.TEST_USERNAME }} + # run: pytest -v --level local tests/test_servers/ + + server-tests-logged-in-level-local: + runs-on: ubuntu-latest + steps: + - name: Check out repository code + uses: actions/checkout@v3 + + - name: Setup Runhouse + uses: ./.github/workflows/setup_runhouse + + - name: Setup ~/.rh/config.yaml + uses: ./.github/workflows/setup_rh_config + with: + username: ${{ secrets.CI_ACCOUNT_USERNAME }} + token: ${{ secrets.CI_ACCOUNT_TOKEN }} + api_server_url: ${{ env.API_SERVER_URL }} + + + - name: pytest -v --level local tests/test_servers/ + env: + TEST_TOKEN: ${{ secrets.TEST_TOKEN }} + TEST_USERNAME: ${{ secrets.TEST_USERNAME }} + run: pytest -v --level local tests/test_servers/ --api-server-url $API_SERVER_URL + timeout-minutes: 60 + + # most-tests-logged-out-level-local: + # runs-on: ubuntu-latest + # steps: + # - name: Check out repository code + # uses: actions/checkout@v3 + + # - name: Setup Runhouse + # uses: ./.github/workflows/setup_runhouse + + # - name: pytest -v --level local -k "not servertest and not secrettest" + # env: + # TEST_TOKEN: ${{ secrets.TEST_TOKEN }} + # TEST_USERNAME: ${{ secrets.TEST_USERNAME }} + # run: pytest -v --level local -k "not servertest and not secrettest" + + most-tests-logged-in-level-local: + runs-on: ubuntu-latest + steps: + - name: Check out repository code + uses: actions/checkout@v3 + + - name: Setup Runhouse + uses: ./.github/workflows/setup_runhouse + + - name: Setup ~/.rh/config.yaml + uses: ./.github/workflows/setup_rh_config + with: + username: ${{ secrets.CI_ACCOUNT_USERNAME }} + token: ${{ secrets.CI_ACCOUNT_TOKEN }} + api_server_url: ${{ env.API_SERVER_URL }} + + - name: pytest -v --level local -k "not servertest and not secrettest and not moduletest and not functiontest and not envtest" + env: + TEST_TOKEN: ${{ secrets.TEST_TOKEN }} + TEST_USERNAME: ${{ secrets.TEST_USERNAME }} + run: pytest -v --level local -k "not servertest and not secrettest and not moduletest and not functiontest and not envtest" --api-server-url $API_SERVER_URL + timeout-minutes: 60 + + # secret-tests-logged-out-level-local: + # runs-on: ubuntu-latest + # steps: + # - name: Check out repository code + # uses: actions/checkout@v3 + + # - name: Setup Runhouse + # uses: ./.github/workflows/setup_runhouse + + # - name: pytest -v --level local -k "secrettest" + # env: + # TEST_TOKEN: ${{ secrets.TEST_TOKEN }} + # TEST_USERNAME: ${{ secrets.TEST_USERNAME }} + # run: pytest -v --level local -k "secrettest" + + secret-tests-logged-in-level-local: + runs-on: ubuntu-latest + steps: + - name: Check out repository code + uses: actions/checkout@v3 + + - name: Setup Runhouse + uses: ./.github/workflows/setup_runhouse + + - name: Setup ~/.rh/config.yaml + uses: ./.github/workflows/setup_rh_config + with: + username: ${{ secrets.CI_ACCOUNT_USERNAME }} + token: ${{ secrets.CI_ACCOUNT_TOKEN }} + api_server_url: ${{ env.API_SERVER_URL }} + + - name: pytest -v --level local -k "secrettest" + env: + TEST_TOKEN: ${{ secrets.TEST_TOKEN }} + TEST_USERNAME: ${{ secrets.TEST_USERNAME }} + run: pytest -v --level local -k "secrettest" --api-server-url $API_SERVER_URL + timeout-minutes: 60 + + module-tests-logged-in-level-local: + runs-on: ubuntu-latest + steps: + - name: Check out repository code + uses: actions/checkout@v3 + + - name: Setup Runhouse + uses: ./.github/workflows/setup_runhouse + + - name: Setup ~/.rh/config.yaml + uses: ./.github/workflows/setup_rh_config + with: + username: ${{ secrets.CI_ACCOUNT_USERNAME }} + token: ${{ secrets.CI_ACCOUNT_TOKEN }} + api_server_url: ${{ env.API_SERVER_URL }} + + - name: pytest -v --level local -k "moduletest" + env: + TEST_TOKEN: ${{ secrets.TEST_TOKEN }} + TEST_USERNAME: ${{ secrets.TEST_USERNAME }} + run: pytest -v --level local -k "moduletest" --api-server-url $API_SERVER_URL + timeout-minutes: 60 + + function-tests-logged-in-level-local: + runs-on: ubuntu-latest + steps: + - name: Check out repository code + uses: actions/checkout@v3 + + - name: Setup Runhouse + uses: ./.github/workflows/setup_runhouse + + - name: Setup ~/.rh/config.yaml + uses: ./.github/workflows/setup_rh_config + with: + username: ${{ secrets.CI_ACCOUNT_USERNAME }} + token: ${{ secrets.CI_ACCOUNT_TOKEN }} + api_server_url: ${{ env.API_SERVER_URL }} + + - name: pytest -v --level local -k "functiontest" + env: + TEST_TOKEN: ${{ secrets.TEST_TOKEN }} + TEST_USERNAME: ${{ secrets.TEST_USERNAME }} + run: pytest -v --level local -k "functiontest" --api-server-url $API_SERVER_URL + timeout-minutes: 60 + + env-tests-logged-in-level-local: + runs-on: ubuntu-latest + steps: + - name: Check out repository code + uses: actions/checkout@v3 + + - name: Setup Runhouse + uses: ./.github/workflows/setup_runhouse + + - name: Setup ~/.rh/config.yaml + uses: ./.github/workflows/setup_rh_config + with: + username: ${{ secrets.CI_ACCOUNT_USERNAME }} + token: ${{ secrets.CI_ACCOUNT_TOKEN }} + api_server_url: ${{ env.API_SERVER_URL }} + + - name: pytest -v --level local -k "envtest" + env: + TEST_TOKEN: ${{ secrets.TEST_TOKEN }} + TEST_USERNAME: ${{ secrets.TEST_USERNAME }} + run: pytest -v --level local -k "envtest" --api-server-url $API_SERVER_URL + timeout-minutes: 60 diff --git a/.github/workflows/setup_rh_config/action.yaml b/.github/workflows/setup_rh_config/action.yaml index 44d6568ba..a344a711b 100644 --- a/.github/workflows/setup_rh_config/action.yaml +++ b/.github/workflows/setup_rh_config/action.yaml @@ -11,6 +11,10 @@ inputs: description: 'The token of the logged in username' required: true + api_server_url: + description: 'The den api server to send the requests to' + required: true + runs: using: composite steps: @@ -22,3 +26,4 @@ runs: echo "disable_data_collection: true" >> ~/.rh/config.yaml echo "token: ${{ inputs.token }}" >> ~/.rh/config.yaml echo "username: ${{ inputs.username }}" >> ~/.rh/config.yaml + echo "api_server_url: ${{ inputs.api_server_url }}" >> ~/.rh/config.yaml diff --git a/runhouse/main.py b/runhouse/main.py index 711e945c6..274965621 100644 --- a/runhouse/main.py +++ b/runhouse/main.py @@ -296,6 +296,7 @@ def _start_server( domain=None, certs_address=None, use_local_telemetry=False, + api_server_url=None, ): ############################################ # Build CLI commands to start the server @@ -369,6 +370,13 @@ def _start_server( logger.info("Configuring local telemetry on the cluster.") flags.append(use_local_telemetry_flag) + api_server_url_flag = ( + f" --api-server-url {api_server_url}" if api_server_url else "" + ) + if api_server_url_flag: + logger.info(f"Setting api_server url to {api_server_url}") + flags.append(api_server_url_flag) + # Check if screen or nohup are available screen = screen and _check_if_command_exists("screen") nohup = not screen and nohup and _check_if_command_exists("nohup") @@ -535,6 +543,10 @@ def restart( False, help="Whether to use local telemetry", ), + api_server_url: str = typer.Option( + default="https://api.run.house", + help="URL of Runhouse Den", + ), ): """Restart the HTTP server on the cluster.""" if name: @@ -559,6 +571,7 @@ def restart( domain=domain, certs_address=certs_address, use_local_telemetry=use_local_telemetry, + api_server_url=api_server_url, ) diff --git a/runhouse/resources/hardware/cluster.py b/runhouse/resources/hardware/cluster.py index b97e7447a..1275018cb 100644 --- a/runhouse/resources/hardware/cluster.py +++ b/runhouse/resources/hardware/cluster.py @@ -113,6 +113,7 @@ def creds_values(self) -> Dict: return self._creds.values def save_config_to_cluster(self, node: str = None): + config = self.config(condensed=False) config.pop("creds") json_config = f"{json.dumps(config)}" @@ -196,6 +197,7 @@ def config(self, condensed=True): creds = creds.replace("loaded_secret_", "") config["creds"] = creds + config["api_server_url"] = rns_client.api_server_url if self._use_custom_certs: config["ssl_certfile"] = self.cert_config.cert_path @@ -743,11 +745,12 @@ def restart_server( + (f" --domain {domain}" if domain else "") + (" --use-local-telemetry" if use_local_telemetry else "") + f" --port {self.server_port}" + + f" --api-server-url {rns_client.api_server_url}" ) status_codes = self.run(commands=[cmd], env=env) if not status_codes[0][0] == 0: - raise ValueError(f"Failed to restart server {self.name}.") + raise ValueError(f"Failed to restart server {self.name}") if https_flag: rns_address = self.rns_address or self.name diff --git a/runhouse/rns/defaults.py b/runhouse/rns/defaults.py index eed6ef4de..a87718065 100644 --- a/runhouse/rns/defaults.py +++ b/runhouse/rns/defaults.py @@ -233,6 +233,11 @@ def set_many(self, key_value_pairs: Dict, config_path: Optional[str] = None): # TODO [DG] allow hierarchical defaults from folders and groups def get(self, key: str, alt: Any = None) -> Any: + # Prioritize env vars + env_var = os.getenv(key.upper()) + if env_var is not None: + return env_var + res = self.defaults_cache.get(key, alt) if not res and key in self.BASE_DEFAULTS: res = self.BASE_DEFAULTS[key] diff --git a/runhouse/rns/rns_client.py b/runhouse/rns/rns_client.py index c54173542..ec5f79d3b 100644 --- a/runhouse/rns/rns_client.py +++ b/runhouse/rns/rns_client.py @@ -153,6 +153,9 @@ def username(self): @property def api_server_url(self): + url_as_env_var = os.getenv("API_SERVER_URL") + if url_as_env_var: + return url_as_env_var return self._configs.get("api_server_url", None) def _index_base_folders(self, lst): diff --git a/runhouse/servers/cluster_servlet.py b/runhouse/servers/cluster_servlet.py index bb3cfbf34..4cae0682b 100644 --- a/runhouse/servers/cluster_servlet.py +++ b/runhouse/servers/cluster_servlet.py @@ -28,7 +28,7 @@ async def __init__( ) self._initialized_env_servlet_names: Set[str] = set() self._key_to_env_servlet_name: Dict[Any, str] = {} - self._auth_cache: AuthCache = AuthCache() + self._auth_cache: AuthCache = AuthCache(cluster_config) if self.cluster_config.get("resource_subtype", None) == "OnDemandCluster": self._last_activity = time.time() diff --git a/runhouse/servers/http/auth.py b/runhouse/servers/http/auth.py index 9cc5d2ec7..06c7ab5e1 100644 --- a/runhouse/servers/http/auth.py +++ b/runhouse/servers/http/auth.py @@ -10,9 +10,10 @@ class AuthCache: # Maps a user's token to all the resources they have access to - def __init__(self): + def __init__(self, cluster_config: dict): self.CACHE = {} self.USERNAMES = {} + self.cluster_config = cluster_config def get_username(self, token: str) -> Optional[str]: """Get username associated with a particular token""" @@ -41,7 +42,7 @@ def lookup_access_level( else: resource_uri_to_send = resource_uri.replace("/", ":") - uri = f"{rns_client.api_server_url}/resource/{resource_uri_to_send}" + uri = f"{self.cluster_config.get('api_server_url')}/resource/{resource_uri_to_send}" resp = rns_client.session.get( uri, headers={"Authorization": f"Bearer {token}"}, diff --git a/runhouse/servers/http/http_server.py b/runhouse/servers/http/http_server.py index 71adf5446..0e4d5fcb6 100644 --- a/runhouse/servers/http/http_server.py +++ b/runhouse/servers/http/http_server.py @@ -81,7 +81,6 @@ async def wrapper(*args, **kwargs): "https://run.house/login to retrieve a token. If calling via HTTP, please " "provide a valid token in the Authorization header.", ) - cluster_uri = (await obj_store.aget_cluster_config()).get("name") cluster_access = await averify_cluster_access(cluster_uri, token) if not cluster_access: @@ -906,10 +905,18 @@ async def main(): help="Address to use for generating self-signed certs and enabling HTTPS. (e.g. public IP address)", ) + parser.add_argument( + "--api-server-url", + type=str, + default=rns_client.api_server_url, + help="URL of Runhouse Den", + ) + parse_args = parser.parse_args() conda_name = parse_args.conda_env restart_proxy = parse_args.restart_proxy + api_server_url = parse_args.api_server_url # The object store and the cluster servlet within it need to be # initialized in order to call `obj_store.get_cluster_config()`, which @@ -936,6 +943,9 @@ async def main(): # cluster_config.json or via CLI args ######################################## + # Den URL + cluster_config["api_server_url"] = api_server_url + # Server port if parse_args.port is not None and parse_args.port != cluster_config.get( "server_port" diff --git a/tests/conftest.py b/tests/conftest.py index 604bea9c8..e2b6385a4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -122,6 +122,13 @@ def pytest_addoption(parser): help="Restart the server on the cluster fixtures.", ) + parser.addoption( + "--api-server-url", + action="store", + default="https://api.run.house", + help="URL of Runhouse Den", + ) + def pytest_generate_tests(metafunc): level = metafunc.config.getoption("level") @@ -160,7 +167,10 @@ def pytest_collection_modifyitems(config, items): items[:] = new_items -def pytest_configure(): +def pytest_configure(config): + import os + + os.environ["API_SERVER_URL"] = config.getoption("api_server_url") pytest.init_args = {} subprocess.run(RAY_START_CMD, shell=True) diff --git a/tests/fixtures/docker_cluster_fixtures.py b/tests/fixtures/docker_cluster_fixtures.py index c83bca6a8..472be0b4f 100644 --- a/tests/fixtures/docker_cluster_fixtures.py +++ b/tests/fixtures/docker_cluster_fixtures.py @@ -13,6 +13,7 @@ import yaml from runhouse.constants import DEFAULT_HTTP_PORT, DEFAULT_HTTPS_PORT, DEFAULT_SSH_PORT +from runhouse.globals import rns_client from tests.conftest import init_args from tests.utils import friend_account @@ -303,10 +304,16 @@ def docker_cluster_pk_tls_exposed(request, test_rns_folder): - Caddy set up on startup to forward Runhouse HTTP server to port 443 - Telemetry enabled """ + import os # From pytest config detached = request.config.getoption("--detached") force_rebuild = request.config.getoption("--force-rebuild") + api_server_url = request.config.getoption("--api-server-url") + + if not api_server_url: + api_server_url = rns_client.api_server_url + os.environ["API_SERVER_URL"] = api_server_url # Ports to use on the Docker VM such that they don't conflict local_ssh_port = BASE_LOCAL_SSH_PORT + 1 @@ -353,10 +360,16 @@ def docker_cluster_pk_ssh(request, test_rns_folder): - Nginx set up on startup to forward Runhouse HTTP server to port 443 - Telemetry enabled """ + import os # From pytest config detached = request.config.getoption("--detached") force_rebuild = request.config.getoption("--force-rebuild") + api_server_url = request.config.getoption("--api-server-url") + + if not api_server_url: + api_server_url = rns_client.api_server_url + os.environ["API_SERVER_URL"] = api_server_url # Ports to use on the Docker VM such that they don't conflict local_ssh_port = BASE_LOCAL_SSH_PORT + 2 @@ -437,10 +450,16 @@ def docker_cluster_pk_http_exposed(request, test_rns_folder): - Caddy set up on startup to forward Runhouse HTTP Server to port 80 - Telemetry enabled """ + import os # From pytest config detached = request.config.getoption("--detached") force_rebuild = request.config.getoption("--force-rebuild") + api_server_url = request.config.getoption("--api-server-url") + + if not api_server_url: + api_server_url = rns_client.api_server_url + os.environ["API_SERVER_URL"] = api_server_url # Ports to use on the Docker VM such that they don't conflict local_ssh_port = BASE_LOCAL_SSH_PORT + 3 @@ -486,10 +505,16 @@ def docker_cluster_pwd_ssh_no_auth(request, test_rns_folder): - No Den Auth - No caddy/port forwarding set up """ + import os # From pytest config detached = request.config.getoption("--detached") force_rebuild = request.config.getoption("--force-rebuild") + api_server_url = request.config.getoption("--api-server-url") + + if not api_server_url: + api_server_url = rns_client.api_server_url + os.environ["API_SERVER_URL"] = api_server_url # Ports to use on the Docker VM such that they don't conflict local_ssh_port = BASE_LOCAL_SSH_PORT + 4 @@ -527,10 +552,17 @@ def friend_account_logged_in_docker_cluster_pk_ssh(request, test_rns_folder): This fixture is not parameterized for every test; it is a separate cluster started with a test account (username: kitchen_tester) in order to test sharing resources with other users. """ + import os # From pytest config detached = request.config.getoption("--detached") force_rebuild = request.config.getoption("--force-rebuild") + api_server_url = request.config.getoption("--api-server-url") + + if not api_server_url: + api_server_url = rns_client.api_server_url + os.environ["API_SERVER_URL"] = api_server_url + with friend_account(): # Ports to use on the Docker VM such that they don't conflict local_ssh_port = BASE_LOCAL_SSH_PORT + 5 diff --git a/tests/test_resources/test_clusters/test_cluster.py b/tests/test_resources/test_clusters/test_cluster.py index e6786f85d..78c1acc42 100644 --- a/tests/test_resources/test_clusters/test_cluster.py +++ b/tests/test_resources/test_clusters/test_cluster.py @@ -320,7 +320,7 @@ def test_condensed_config_for_cluster(self, cluster): on_cluster_config = remote_cluster_config() local_cluster_config = cluster.config() - keys_to_skip = ["creds", "client_port", "server_host"] + keys_to_skip = ["creds", "client_port", "server_host", "api_server_url"] on_cluster_config = remove_config_keys(on_cluster_config, keys_to_skip) local_cluster_config = remove_config_keys(local_cluster_config, keys_to_skip)