Skip to content

Commit

Permalink
Clean up an EnvServlet from global Runhouse daemon state if it dies
Browse files Browse the repository at this point in the history
unexpectedly.
  • Loading branch information
rohinb2 committed Apr 15, 2024
1 parent 575e711 commit 18012af
Showing 1 changed file with 13 additions and 1 deletion.
14 changes: 13 additions & 1 deletion runhouse/servers/obj_store.py
Original file line number Diff line number Diff line change
Expand Up @@ -264,7 +264,19 @@ def initialize(
@staticmethod
async def acall_env_servlet_method(servlet_name: str, method: str, *args, **kwargs):
env_servlet = ObjStore.get_env_servlet(servlet_name)
return await ObjStore.acall_actor_method(env_servlet, method, *args, **kwargs)
try:
return await ObjStore.acall_actor_method(
env_servlet, method, *args, **kwargs
)
except ray.exceptions.RayActorError as e:
if "died unexpectedly before finishing this task" in str(e):
# Need to clean up the env_servlet everywhere
# Need an actual instance of the ObjStore that has a cluster servlet set up in order to do this
from runhouse.globals import obj_store

await obj_store.adelete_env_contents(servlet_name)

raise e

@staticmethod
async def acall_actor_method(
Expand Down

0 comments on commit 18012af

Please sign in to comment.