From 18012af6591b17c5c011c3c0437120f4965ed1f4 Mon Sep 17 00:00:00 2001 From: Rohin Bhasin Date: Mon, 15 Apr 2024 14:39:57 -0400 Subject: [PATCH] Clean up an `EnvServlet` from global Runhouse daemon state if it dies unexpectedly. --- runhouse/servers/obj_store.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/runhouse/servers/obj_store.py b/runhouse/servers/obj_store.py index a2fe3ff6d..3afef9772 100644 --- a/runhouse/servers/obj_store.py +++ b/runhouse/servers/obj_store.py @@ -264,7 +264,19 @@ def initialize( @staticmethod async def acall_env_servlet_method(servlet_name: str, method: str, *args, **kwargs): env_servlet = ObjStore.get_env_servlet(servlet_name) - return await ObjStore.acall_actor_method(env_servlet, method, *args, **kwargs) + try: + return await ObjStore.acall_actor_method( + env_servlet, method, *args, **kwargs + ) + except ray.exceptions.RayActorError as e: + if "died unexpectedly before finishing this task" in str(e): + # Need to clean up the env_servlet everywhere + # Need an actual instance of the ObjStore that has a cluster servlet set up in order to do this + from runhouse.globals import obj_store + + await obj_store.adelete_env_contents(servlet_name) + + raise e @staticmethod async def acall_actor_method(