From 31b8fb9b486ee6db93bef125da8197fcb1e328d1 Mon Sep 17 00:00:00 2001 From: YuriCat Date: Sat, 26 Feb 2022 03:45:40 +0900 Subject: [PATCH 01/10] chore: remove fileno() inferface from PickledConnection --- handyrl/connection.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/handyrl/connection.py b/handyrl/connection.py index 3550f153..8880470b 100755 --- a/handyrl/connection.py +++ b/handyrl/connection.py @@ -29,9 +29,6 @@ def close(self): self.conn.close() self.conn = None - def fileno(self): - return self.conn.fileno() - def _recv(self, size): buf = io.BytesIO() while size > 0: From 58737c61f325340ef610822d39a9bb06e92929d9 Mon Sep 17 00:00:00 2001 From: YuriCat Date: Wed, 6 Jul 2022 23:21:14 +0900 Subject: [PATCH 02/10] feature: add python3.10 test --- .github/workflows/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/action.yaml b/.github/workflows/action.yaml index 4b511d92..698e729f 100644 --- a/.github/workflows/action.yaml +++ b/.github/workflows/action.yaml @@ -12,7 +12,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: [3.7, 3.8, 3.9] + python-version: ['3.7', '3.8', '3.9', '3.10'] steps: - name: Checkout uses: actions/checkout@v2 From 875313dea181bbf563c292acac7ae893b781fa2f Mon Sep 17 00:00:00 2001 From: YuriCat Date: Thu, 22 Dec 2022 22:56:53 +0900 Subject: [PATCH 03/10] feature: return dict from evaluation function (same key) --- handyrl/evaluation.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/handyrl/evaluation.py b/handyrl/evaluation.py index 2d391bce..4ffe0192 100755 --- a/handyrl/evaluation.py +++ b/handyrl/evaluation.py @@ -106,7 +106,7 @@ def exec_match(env, agents, critic=None, show=False, game_args={}): outcome = env.outcome() if show: print('final outcome = %s' % outcome) - return outcome + return {'result': outcome} def exec_network_match(env, network_agents, critic=None, show=False, game_args={}): @@ -138,7 +138,7 @@ def exec_network_match(env, network_agents, critic=None, show=False, game_args={ outcome = env.outcome() for p, agent in network_agents.items(): agent.outcome(outcome[p]) - return outcome + return {'result': outcome} def build_agent(raw, env=None): @@ -170,11 +170,11 @@ def execute(self, models, args): else: agents[p] = Agent(model) - outcome = exec_match(self.env, agents) - if outcome is None: + results = exec_match(self.env, agents) + if results is None: print('None episode in evaluation!') return None - return {'args': args, 'result': outcome, 'opponent': opponent} + return {'args': args, 'opponent': opponent, **results} def wp_func(results): @@ -196,10 +196,10 @@ def eval_process_mp_child(agents, critic, env_args, index, in_queue, out_queue, print('*** Game %d ***' % g) agent_map = {env.players()[p]: agents[ai] for p, ai in enumerate(agent_ids)} if isinstance(list(agent_map.values())[0], NetworkAgent): - outcome = exec_network_match(env, agent_map, critic, show=show, game_args=game_args) + results = exec_network_match(env, agent_map, critic, show=show, game_args=game_args) else: - outcome = exec_match(env, agent_map, critic, show=show, game_args=game_args) - out_queue.put((pat_idx, agent_ids, outcome)) + results = exec_match(env, agent_map, critic, show=show, game_args=game_args) + out_queue.put((pat_idx, agent_ids, results)) out_queue.put(None) @@ -246,7 +246,8 @@ def evaluate_mp(env, agents, critic, env_args, args_patterns, num_process, num_g if ret is None: finished_cnt += 1 continue - pat_idx, agent_ids, outcome = ret + pat_idx, agent_ids, results = ret + outcome = results.get('outcome') if outcome is not None: for idx, p in enumerate(env.players()): agent_id = agent_ids[idx] From 50dcefdb020f607a3e1ec5a128ed98a21294ad65 Mon Sep 17 00:00:00 2001 From: YuriCat Date: Thu, 22 Dec 2022 23:46:02 +0900 Subject: [PATCH 04/10] fix: output dict key outcome -> result --- handyrl/evaluation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/handyrl/evaluation.py b/handyrl/evaluation.py index 4ffe0192..45c4d225 100755 --- a/handyrl/evaluation.py +++ b/handyrl/evaluation.py @@ -247,7 +247,7 @@ def evaluate_mp(env, agents, critic, env_args, args_patterns, num_process, num_g finished_cnt += 1 continue pat_idx, agent_ids, results = ret - outcome = results.get('outcome') + outcome = results.get('result') if outcome is not None: for idx, p in enumerate(env.players()): agent_id = agent_ids[idx] From 3b375e1c1845701a84216eaf888701053c7d2710 Mon Sep 17 00:00:00 2001 From: YuriCat Date: Thu, 5 Jan 2023 18:27:15 +0900 Subject: [PATCH 05/10] feature: data gathering code both for solo/multi player training --- handyrl/train.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/handyrl/train.py b/handyrl/train.py index dd0b257c..f4c75ade 100755 --- a/handyrl/train.py +++ b/handyrl/train.py @@ -63,15 +63,14 @@ def replace_none(a, b): # data that is changed by training configuration if args['turn_based_training'] and not args['observation']: - obs = [[m['observation'][m['turn'][0]]] for m in moments] - prob = np.array([[[m['selected_prob'][m['turn'][0]]]] for m in moments]) - act = np.array([[m['action'][m['turn'][0]]] for m in moments], dtype=np.int64)[..., np.newaxis] - amask = np.array([[m['action_mask'][m['turn'][0]]] for m in moments]) + players_list = [[m['turn'][0]] for m in moments] else: - obs = [[replace_none(m['observation'][player], obs_zeros) for player in players] for m in moments] - prob = np.array([[[replace_none(m['selected_prob'][player], 1.0)] for player in players] for m in moments]) - act = np.array([[replace_none(m['action'][player], 0) for player in players] for m in moments], dtype=np.int64)[..., np.newaxis] - amask = np.array([[replace_none(m['action_mask'][player], amask_zeros + 1e32) for player in players] for m in moments]) + players_list = [players for m in moments] + + obs = [[replace_none(m['observation'][player], obs_zeros) for player in players_] for m, players_ in zip(moments, players_list)] + prob = np.array([[[replace_none(m['selected_prob'][player], 1.0)] for player in players_] for m, players_ in zip(moments, players_list)]) + act = np.array([[replace_none(m['action'][player], 0) for player in players_] for m, players_ in zip(moments, players_list)], dtype=np.int64)[..., np.newaxis] + amask = np.array([[replace_none(m['action_mask'][player], amask_zeros + 1e32) for player in players_] for m, players_ in zip(moments, players_list)]) # reshape observation obs = rotate(rotate(obs)) # (T, P, ..., ...) -> (P, ..., T, ...) -> (..., T, P, ...) From 98b595d7f7d7777fc8fad25f77cab07bf27c0fc6 Mon Sep 17 00:00:00 2001 From: YuriCat Date: Tue, 18 Apr 2023 19:42:59 +0900 Subject: [PATCH 06/10] fix: fill 0 for reward, return, value in make_batch() --- handyrl/train.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/handyrl/train.py b/handyrl/train.py index 5f8c43ae..7dad705f 100755 --- a/handyrl/train.py +++ b/handyrl/train.py @@ -77,9 +77,9 @@ def replace_none(a, b): obs = bimap_r(obs_zeros, obs, lambda _, o: np.array(o)) # datum that is not changed by training configuration - v = np.array([[replace_none(m['value'][player], [0]) for player in players] for m in moments], dtype=np.float32).reshape(len(moments), len(players), -1) - rew = np.array([[replace_none(m['reward'][player], [0]) for player in players] for m in moments], dtype=np.float32).reshape(len(moments), len(players), -1) - ret = np.array([[replace_none(m['return'][player], [0]) for player in players] for m in moments], dtype=np.float32).reshape(len(moments), len(players), -1) + v = np.array([[replace_none(m['value'][player], 0) for player in players] for m in moments], dtype=np.float32).reshape(len(moments), len(players), -1) + rew = np.array([[replace_none(m['reward'][player], 0) for player in players] for m in moments], dtype=np.float32).reshape(len(moments), len(players), -1) + ret = np.array([[replace_none(m['return'][player], 0) for player in players] for m in moments], dtype=np.float32).reshape(len(moments), len(players), -1) oc = np.array([ep['outcome'][player] for player in players], dtype=np.float32).reshape(1, len(players), -1) emask = np.ones((len(moments), 1, 1), dtype=np.float32) # episode mask From c5472e2e60315a7d045e5f82f8bcf7af3a1a0f1e Mon Sep 17 00:00:00 2001 From: YuriCat Date: Tue, 18 Apr 2023 19:44:50 +0900 Subject: [PATCH 07/10] feature: remove installation of kaggle environments from githb actions --- .github/workflows/action.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/action.yaml b/.github/workflows/action.yaml index 698e729f..28cbe9ca 100644 --- a/.github/workflows/action.yaml +++ b/.github/workflows/action.yaml @@ -24,7 +24,6 @@ jobs: run: | python -m pip install --upgrade pip pip install -r requirements.txt - pip install -r handyrl/envs/kaggle/requirements.txt - name: pytest run: | python -m pytest tests From 32018a564505c011abe597fb43a0668122bb1151 Mon Sep 17 00:00:00 2001 From: YuriCat Date: Tue, 18 Apr 2023 19:50:14 +0900 Subject: [PATCH 08/10] feature: remove hungry_geese from environment test --- tests/test_environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_environment.py b/tests/test_environment.py index 14ff9f64..5f0e9c23 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -8,7 +8,7 @@ 'tictactoe', 'geister', 'parallel_tictactoe', - 'kaggle.hungry_geese', + # 'kaggle.hungry_geese', ] From 2b4beeed758791e53b2e53c3b6811d79ced2f133 Mon Sep 17 00:00:00 2001 From: YuriCat Date: Fri, 22 Sep 2023 22:08:26 +0900 Subject: [PATCH 09/10] feature: remove Python 3.7 from github CI actions --- .github/workflows/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/action.yaml b/.github/workflows/action.yaml index 698e729f..a066a8fc 100644 --- a/.github/workflows/action.yaml +++ b/.github/workflows/action.yaml @@ -12,7 +12,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: ['3.7', '3.8', '3.9', '3.10'] + python-version: ['3.8', '3.9', '3.10'] steps: - name: Checkout uses: actions/checkout@v2 From 0becf95dfe1d4eef43af17da99e8ecb92595aa79 Mon Sep 17 00:00:00 2001 From: YuriCat Date: Tue, 21 Nov 2023 22:59:37 +0900 Subject: [PATCH 10/10] Revert "chore: remove fileno() inferface from PickledConnection" This reverts commit 31b8fb9b486ee6db93bef125da8197fcb1e328d1. --- handyrl/connection.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/handyrl/connection.py b/handyrl/connection.py index 8b01d555..49a176ee 100755 --- a/handyrl/connection.py +++ b/handyrl/connection.py @@ -29,6 +29,9 @@ def close(self): self.conn.close() self.conn = None + def fileno(self): + return self.conn.fileno() + def _recv(self, size): buf = io.BytesIO() while size > 0: