Tianshou_Training_GoDot_PPO_OldCol.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "invalid syntax (collector.py, line 324)",
     "output_type": "error",
     "traceback": [
      "Traceback \u001b[1;36m(most recent call last)\u001b[0m:\n",
      "\u001b[0m  File \u001b[0;32md:\\Projects\\B-ACE\\b_ace_env_tianshou\\Lib\\site-packages\\IPython\\core\\interactiveshell.py:3577\u001b[0m in \u001b[0;35mrun_code\u001b[0m\n    exec(code_obj, self.user_global_ns, self.user_ns)\u001b[0m\n",
      "\u001b[0m  Cell \u001b[0;32mIn[1], line 11\u001b[0m\n    from tianshou.data import Collector, VectorReplayBuffer, PrioritizedVectorReplayBuffer\u001b[0m\n",
      "\u001b[0m  File \u001b[0;32md:\\Projects\\B-ACE\\b_ace_env_tianshou\\Lib\\site-packages\\tianshou\\__init__.py:1\u001b[0m\n    from tianshou import data, env, exploration, policy, trainer, utils\u001b[0m\n",
      "\u001b[1;36m  File \u001b[1;32md:\\Projects\\B-ACE\\b_ace_env_tianshou\\Lib\\site-packages\\tianshou\\data\\__init__.py:27\u001b[1;36m\n\u001b[1;33m    from tianshou.data.collector import Collector, AsyncCollector, CollectStats, CollectStatsBase\u001b[1;36m\n",
      "\u001b[1;36m  File \u001b[1;32md:\\Projects\\B-ACE\\b_ace_env_tianshou\\Lib\\site-packages\\tianshou\\data\\collector.py:324\u001b[1;36m\u001b[0m\n\u001b[1;33m    elapsed_time_ms =\u001b[0m\n\u001b[1;37m                      ^\u001b[0m\n\u001b[1;31mSyntaxError\u001b[0m\u001b[1;31m:\u001b[0m invalid syntax\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import datetime\n",
    "from typing import Optional, Tuple\n",
    "import json\n",
    "import numpy as np\n",
    "import torch\n",
    "from gymnasium.spaces import Box, Discrete\n",
    "\n",
    "from torch.distributions import Normal, Distribution\n",
    "\n",
    "from tianshou.data import Collector, VectorReplayBuffer, PrioritizedVectorReplayBuffer\n",
    "from tianshou.env import DummyVectorEnv\n",
    "from tianshou.env.pettingzoo_env_parallel import PettingZooParallelEnv\n",
    "from tianshou.env.pettingzoo_env import PettingZooEnv\n",
    "\n",
    "#from PettingZooParallelEnv import PettingZooParallelEnv\n",
    "\n",
    "\n",
    "from tianshou.policy import PPOPolicy\n",
    "from tianshou.trainer import OnpolicyTrainer\n",
    "\n",
    "from tianshou.utils.net.common import ActorCritic, DataParallelNet, Net\n",
    "from tianshou.utils.net.continuous import Actor, Critic\n",
    "\n",
    "from tianshou.policy import BasePolicy, DQNPolicy, MultiAgentPolicyManager, RandomPolicy, RainbowPolicy\n",
    "from tianshou.trainer import OffpolicyTrainer\n",
    "from torch.utils.tensorboard import SummaryWriter\n",
    "from DNN_B_ACE_ACTOR import DNN_B_ACE_ACTOR\n",
    "from DNN_B_ACE_CRITIC import DNN_B_ACE_CRITIC\n",
    "from GodotRLPettingZooWrapper import GodotRLPettingZooWrapper\n",
    "\n",
    "from CollectorMA import CollectorMA\n",
    "from MAParalellPolicy import MAParalellPolicy\n",
    "\n",
    "\n",
    "####---------------------------#######\n",
    "#Tianshou Adjustment\n",
    "import wandb\n",
    "# os.environ[\"WANDB_NOTEBOOK_NAME\"] = \"Tianshow_Training_GoDot.ipybn\"\n",
    "from tianshou.utils import WandbLogger\n",
    "# from tianshou.utils.logger.base import LOG_DATA_TYPE\n",
    "# def new_write(self, step_type: str, step: int, data: LOG_DATA_TYPE) -> None:\n",
    "#      data[step_type] = step\n",
    "#      wandb.log(data)   \n",
    "# WandbLogger.write = new_write \n",
    "####---------------------------#######\n",
    "\n",
    "\n",
    "model  =  \"PPO_DNN\"#\"SISL_Task_MultiHead\" #\"CNN_ATT_SISL\" #\"MultiHead_SISL\" \n",
    "test_num  =  \"_B_ACE01\"\n",
    "policyModel  =  \"PPO\"\n",
    "name = model + test_num\n",
    "\n",
    "train_env_num = 10\n",
    "test_env_num = 10\n",
    "\n",
    "now = datetime.datetime.now().strftime(\"%y%m%d-%H%M%S\")\n",
    "log_name = name + str(now)\n",
    "log_path = os.path.join('./', \"Logs\", \"dqn_sisl\", log_name)\n",
    "\n",
    "load_policy_name = f'policy_SISL_Task_MultiHead_Desk_NewExpCor231219-173711_44.pth'\n",
    "save_policy_name = f'policy_{log_name}'\n",
    "policy_path = \"ppo_B_ACE\"\n",
    "\n",
    "\n",
    "model_load_path = os.path.join(policy_path, load_policy_name)  \n",
    "model_save_path = os.path.join(policy_path, save_policy_name)        \n",
    "os.makedirs(os.path.join(policy_path), exist_ok=True)  \n",
    "os.makedirs(os.path.join(log_path), exist_ok=True)\n",
    "\n",
    "Policy_Config = {\n",
    "    \"same_policy\" : True,\n",
    "    \"load_model\" : False,\n",
    "    \"freeze_CNN\" : False     \n",
    "                }\n",
    "\n",
    "B_ACE_Config = { \t\n",
    "                    \"EnvConfig\" : \n",
    "                    {\n",
    "                        \"task\": \"b_ace_v1\",\n",
    "                        \"env_path\": \"BVR_AirCombat/bin/B_ACE_v6.console.exe\",\n",
    "                        \"port\": 12500,\n",
    "                        \"renderize\": 0,\n",
    "                        \"debug_view\": 0,\n",
    "                        \"phy_fps\": 20,\n",
    "                        \"speed_up\": 50000,\n",
    "                        \"max_cycles\": 36000,\n",
    "                        \"experiment_mode\"  : 0,\n",
    "                        \"parallel_envs\": 1,\t\n",
    "                        \"seed\": 1,\t\n",
    "                        \"action_repeat\": 20,\t\n",
    "                        \"action_type\": \"Low_Level_Continuous\",                        \n",
    "                        \"full_observation\": 0,\n",
    "                        \n",
    "                        \"RewardsConfig\" : {\n",
    "                            \"mission_factor\": 1.0,\n",
    "                            \"missile_fire_factor\": -0.1,\n",
    "                            \"missile_no_fire_factor\": -0.001,\n",
    "                            \"missile_miss_factor\": -0.5,\n",
    "                            \"detect_loss_factor\": -0.1,\n",
    "                            \"keep_track_factor\": 0.005,\n",
    "                            \"hit_enemy_factor\": 3.0,\n",
    "                            \"hit_own_factor\": -5.0,\n",
    "                            \"situation_factor\": 0.1,\n",
    "                            \"final_team_killed_factor\": -5.0,\n",
    "                            \"final_enemy_on_target_factor\": -3.0,\n",
    "                            \"final_enemies_killed_factor\": 5.0,\n",
    "                            \"final_max_cycles_factor\": 3.0\n",
    "                        }\n",
    "                    },\n",
    "\n",
    "                    \"AgentsConfig\" : \n",
    "                    {\n",
    "                        \"blue_agents\": { \n",
    "                            \"num_agents\" : 1,\n",
    "                            \"beh_config\" : {\n",
    "                                \"dShot\" : 0.85,\n",
    "                                \"lCrank\": 0.60,\n",
    "                                \"lBreak\": 0.95\n",
    "                            },\n",
    "                            \"base_behavior\": \"external\",                  \n",
    "                            \"init_position\": {\"x\": 0.0, \"y\": 25000.0,\"z\": 30.0},\n",
    "                            \"offset_pos\": {\t\"x\": 0.0, \"y\": 0.0, \"z\": 0.0},\n",
    "                            \"init_hdg\": 0.0,                        \n",
    "                            \"target_position\": {\"x\": 0.0,\"y\": 25000.0,\"z\": 30.0},\n",
    "                            \"rnd_offset_range\":{\"x\": 10.0,\"y\": 10000.0,\"z\": 5.0},\t\t\t\t\n",
    "                            \"rnd_shot_dist_var\": 0.0,\n",
    "                            \"wez_models\" : \"res://assets/Default_Wez_params.json\"\n",
    "                        },\t\n",
    "                        \"red_agents\":\n",
    "                        { \n",
    "                            \"num_agents\" : 1, \n",
    "                            \"base_behavior\": \"baseline1\",\n",
    "                            \"beh_config\" : {\n",
    "                                \"dShot\" : 0.85,\n",
    "                                \"lCrank\": 0.60,\n",
    "                                \"lBreak\": 0.95\n",
    "                            },\n",
    "                            \"init_position\": {\"x\": 0.0,\"y\": 25000.0,\"z\": -30.0},\n",
    "                            \"offset_pos\": {\"x\": 0.0,\"y\": 0.0,\"z\": 0.0},\n",
    "                            \"init_hdg\" : 180.0,                        \n",
    "                            \"target_position\": {\"x\": 0.0,\"y\": 25000.0,\"z\": 30.0},\n",
    "                            \"rnd_offset_range\":{\"x\": 10.0,\"y\": 10000.0,\"z\": 5.0},\t\t\t\t\n",
    "                            \"rnd_shot_dist_var\": 0.0,\n",
    "                            \"wez_models\" : \"res://assets/Default_Wez_params.json\"\n",
    "                        }\n",
    "                    }\t\n",
    "                }\n",
    "#max_cycles = B_ACE_Config[\"max_cycles\"]\n",
    "n_agents = 1#B_ACE_Config[\"n_pursuers\"]\n",
    "\n",
    "dqn_params = {\"discount_factor\": 0.98, \n",
    "              \"estimation_step\": 20, \n",
    "              \"target_update_freq\": 3000,#max_cycles * n_agents,\n",
    "              \"optminizer\": \"Adam\",\n",
    "              \"lr\": 0.00016 }\n",
    "\n",
    "trainer_params = {\"max_epoch\": 500,\n",
    "                  \"step_per_epoch\": 10000,#5 * (150 * n_agents),\n",
    "                  \"step_per_collect\": 1000,# * (10 * n_agents),\n",
    "                  \"episode_per_test\": 20,\n",
    "                  \"batch_size\" : 64 * n_agents,\n",
    "                  \"update_per_step\": 1 / 100, #Only run after close a Collect (run many times as necessary to meet the value)\n",
    "                  \"tn_eps_max\": 0.95,\n",
    "                  \"ts_eps_max\": 0.01,\n",
    "                  \"warmup_size\" : 0\n",
    "                  }\n",
    "\n",
    "runConfig = dqn_params\n",
    "runConfig.update(Policy_Config)\n",
    "runConfig.update(trainer_params) \n",
    "runConfig.update(B_ACE_Config)\n",
    "\n",
    "\n",
    "def _get_agents(\n",
    "    agent_learn: Optional[BasePolicy] = None,\n",
    "    agent_opponent: Optional[BasePolicy] = None,\n",
    "    optim: Optional[torch.optim.Optimizer] = None,\n",
    "    policy_load_path = None,\n",
    ") -> Tuple[BasePolicy, torch.optim.Optimizer, list]:\n",
    "    \n",
    "    env = _get_env()       \n",
    "    agent_observation_space = env.observation_space(\"agent_0\")\n",
    "   \n",
    "    #print(env.action_space)\n",
    "    action_shape = 50#env.action_space.shape\n",
    "    \n",
    "    print(\"ActionSPACE: \", env.action_space())\n",
    "    device=\"cuda\" if torch.cuda.is_available() else \"cpu\"  \n",
    "\n",
    "    agents = []        \n",
    "    \n",
    "    if Policy_Config[\"same_policy\"]:\n",
    "        policies_number = 1\n",
    "    else:\n",
    "        policies_number = 2#len(env.agents)\n",
    "\n",
    "    for _ in range(policies_number):      \n",
    "        \n",
    "        if model == \"DNN_B_ACE\":\n",
    "            net = DNN_B_ACE(\n",
    "                obs_shape=agent_observation_space,                \n",
    "                action_shape=4,                \n",
    "                device=\"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
    "                \n",
    "            ).to(device)        \n",
    "                   \n",
    "        if policyModel == \"DQN\":\n",
    "            agent_learn = DQNPolicy(\n",
    "                model=net,\n",
    "                optim=optim,\n",
    "                action_space = action_shape,\n",
    "                discount_factor= dqn_params[\"discount_factor\"],\n",
    "                estimation_step=dqn_params[\"estimation_step\"],\n",
    "                target_update_freq=dqn_params[\"target_update_freq\"],\n",
    "                reward_normalization = False,\n",
    "                clip_loss_grad = False \n",
    "            ) \n",
    "         \n",
    "         \n",
    "        if model == \"PPO_DNN\":\n",
    "            \n",
    "            actor = DNN_B_ACE_ACTOR(\n",
    "                obs_shape=agent_observation_space.shape,                \n",
    "                action_shape=4,                \n",
    "                device=\"cuda\" if torch.cuda.is_available() else \"cpu\"                \n",
    "            ).to(device)\n",
    "\n",
    "            critic = DNN_B_ACE_CRITIC(\n",
    "                obs_shape=agent_observation_space.shape,                \n",
    "                action_shape=4,                \n",
    "                device=\"cuda\" if torch.cuda.is_available() else \"cpu\"                \n",
    "            ).to(device)\n",
    "            \n",
    "                                    \n",
    "            actor_critic = ActorCritic(actor, critic)\n",
    "        \n",
    "            # orthogonal initialization\n",
    "            # for m in actor_critic.modules():\n",
    "            #     if isinstance(m, torch.nn.Linear):\n",
    "            #         torch.nn.init.orthogonal_(m.weight)\n",
    "            #         torch.nn.init.zeros_(m.bias)            \n",
    "            \n",
    "            # dist = torch.distributions.Normal(torch.tensor([0.0]), torch.tensor([1.0])) \n",
    "                # define policy\n",
    "            def dist(mu, sigma) -> Distribution:\n",
    "                return Normal(mu, sigma)        \n",
    "                \n",
    "            #optim_actor  = torch.optim.Adam(netActor.parameters(),  lr=dqn_params[\"lr\"], weight_decay=0.0, amsgrad= True )\n",
    "            #optim_critic = torch.optim.Adam(netCritic.parameters(), lr=dqn_params[\"lr\"], weight_decay=0.0, amsgrad= True )\n",
    "            optim = torch.optim.Adam(actor_critic.parameters(), lr=dqn_params[\"lr\"])\n",
    "                    \n",
    "            agent_learn = PPOPolicy(\n",
    "                actor=actor,\n",
    "                critic=critic,\n",
    "                optim=optim,\n",
    "                dist_fn=dist,\n",
    "                action_scaling=True,#isinstance(Discrete(50), Box),\n",
    "                discount_factor=0.99,\n",
    "                max_grad_norm= None, #0.5,\n",
    "                eps_clip=0.2,\n",
    "                vf_coef=0.5,\n",
    "                ent_coef=0.01,\n",
    "                gae_lambda=0.95,\n",
    "                reward_normalization=False, #\n",
    "                dual_clip=None,\n",
    "                value_clip=0,\n",
    "                action_space=env.action_space(),\n",
    "                deterministic_eval=False,\n",
    "                advantage_normalization=False,\n",
    "                recompute_advantage=False,\n",
    "                action_bound_method = \"clip\", #Literal[\"clip\", \"tanh\"] | None = \"clip\",\n",
    "                lr_scheduler = None, # TLearningRateScheduler | None = None,\n",
    "            )\n",
    "            \n",
    "            if Policy_Config[\"load_model\"] is True:\n",
    "                # Load the saved checkpoint             \n",
    "                agent_learn.load_state_dict(torch.load(model_load_path))\n",
    "                print(f'Loaded-> {model_load_path}')\n",
    "                   \n",
    "        \n",
    "        agents.append(agent_learn)\n",
    "\n",
    "    if Policy_Config[\"same_policy\"]:\n",
    "        agents = [agents[0] for _ in range(len(env.agents))]\n",
    "    else:\n",
    "        for _ in range(len(env.agents) - policies_number):\n",
    "            agents.append(agents[0])\n",
    "\n",
    "    policy = MultiAgentPolicyManager(policies = agents, env=env)  \n",
    "    #policy = MAParalellPolicy(policies = agents, env=env, device=\"cuda\" if torch.cuda.is_available() else \"cpu\" )  \n",
    "        \n",
    "    return policy, optim, env.agents\n",
    "\n",
    "def _get_env():\n",
    "    \"\"\"This function is needed to provide callables for DummyVectorEnv.\"\"\"   \n",
    "    \n",
    "    \n",
    "    env = GodotRLPettingZooWrapper( convert_action_space = True,\n",
    "                                    device = 'cpu',\n",
    "                                    **B_ACE_Config)\n",
    "    \n",
    "    #env = PettingZooEnv(env)  \n",
    "    \n",
    "    return env  \n",
    "   \n",
    "\n",
    "# print(json.dumps(runConfig, indent=4))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "waiting for remote GODOT connection on port 13562\n",
      "connection established\n",
      "action space {'input': {'action_type': 'continuous', 'size': 4}}\n",
      "observation space {'obs': {'size': [20], 'space': 'box'}}\n",
      "waiting for remote GODOT connection on port 13746\n",
      "connection established\n",
      "action space {'input': {'action_type': 'continuous', 'size': 4}}\n",
      "observation space {'obs': {'size': [20], 'space': 'box'}}\n",
      "waiting for remote GODOT connection on port 11927\n",
      "connection established\n",
      "action space {'input': {'action_type': 'continuous', 'size': 4}}\n",
      "observation space {'obs': {'size': [20], 'space': 'box'}}\n",
      "waiting for remote GODOT connection on port 11817\n",
      "connection established\n",
      "action space {'input': {'action_type': 'continuous', 'size': 4}}\n",
      "observation space {'obs': {'size': [20], 'space': 'box'}}\n",
      "waiting for remote GODOT connection on port 11367\n",
      "connection established\n",
      "action space {'input': {'action_type': 'continuous', 'size': 4}}\n",
      "observation space {'obs': {'size': [20], 'space': 'box'}}\n",
      "waiting for remote GODOT connection on port 12432\n",
      "connection established\n",
      "action space {'input': {'action_type': 'continuous', 'size': 4}}\n",
      "observation space {'obs': {'size': [20], 'space': 'box'}}\n",
      "waiting for remote GODOT connection on port 13669\n",
      "connection established\n",
      "action space {'input': {'action_type': 'continuous', 'size': 4}}\n",
      "observation space {'obs': {'size': [20], 'space': 'box'}}\n",
      "waiting for remote GODOT connection on port 13260\n",
      "connection established\n",
      "action space {'input': {'action_type': 'continuous', 'size': 4}}\n",
      "observation space {'obs': {'size': [20], 'space': 'box'}}\n",
      "waiting for remote GODOT connection on port 11230\n",
      "connection established\n",
      "action space {'input': {'action_type': 'continuous', 'size': 4}}\n",
      "observation space {'obs': {'size': [20], 'space': 'box'}}\n",
      "waiting for remote GODOT connection on port 12744\n",
      "connection established\n",
      "action space {'input': {'action_type': 'continuous', 'size': 4}}\n",
      "observation space {'obs': {'size': [20], 'space': 'box'}}\n",
      "waiting for remote GODOT connection on port 12211\n",
      "connection established\n",
      "action space {'input': {'action_type': 'continuous', 'size': 4}}\n",
      "observation space {'obs': {'size': [20], 'space': 'box'}}\n",
      "waiting for remote GODOT connection on port 12024\n",
      "connection established\n",
      "action space {'input': {'action_type': 'continuous', 'size': 4}}\n",
      "observation space {'obs': {'size': [20], 'space': 'box'}}\n",
      "waiting for remote GODOT connection on port 11555\n",
      "connection established\n",
      "action space {'input': {'action_type': 'continuous', 'size': 4}}\n",
      "observation space {'obs': {'size': [20], 'space': 'box'}}\n",
      "waiting for remote GODOT connection on port 13634\n",
      "connection established\n",
      "action space {'input': {'action_type': 'continuous', 'size': 4}}\n",
      "observation space {'obs': {'size': [20], 'space': 'box'}}\n",
      "waiting for remote GODOT connection on port 13020\n",
      "connection established\n",
      "action space {'input': {'action_type': 'continuous', 'size': 4}}\n",
      "observation space {'obs': {'size': [20], 'space': 'box'}}\n",
      "waiting for remote GODOT connection on port 11155\n",
      "connection established\n",
      "action space {'input': {'action_type': 'continuous', 'size': 4}}\n",
      "observation space {'obs': {'size': [20], 'space': 'box'}}\n",
      "waiting for remote GODOT connection on port 12807\n",
      "connection established\n",
      "action space {'input': {'action_type': 'continuous', 'size': 4}}\n",
      "observation space {'obs': {'size': [20], 'space': 'box'}}\n",
      "waiting for remote GODOT connection on port 13084\n",
      "connection established\n",
      "action space {'input': {'action_type': 'continuous', 'size': 4}}\n",
      "observation space {'obs': {'size': [20], 'space': 'box'}}\n",
      "waiting for remote GODOT connection on port 11014\n",
      "connection established\n",
      "action space {'input': {'action_type': 'continuous', 'size': 4}}\n",
      "observation space {'obs': {'size': [20], 'space': 'box'}}\n",
      "waiting for remote GODOT connection on port 11268\n",
      "connection established\n",
      "action space {'input': {'action_type': 'continuous', 'size': 4}}\n",
      "observation space {'obs': {'size': [20], 'space': 'box'}}\n",
      "waiting for remote GODOT connection on port 11573\n",
      "connection established\n",
      "action space {'input': {'action_type': 'continuous', 'size': 4}}\n",
      "observation space {'obs': {'size': [20], 'space': 'box'}}\n",
      "ActionSPACE:  Box(-1.0, 1.0, (4,), float32)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\Projects\\B-ACE\\b_ace_env_tianshou\\Lib\\site-packages\\torch\\nn\\init.py:452: UserWarning: Initializing zero-element tensors is a no-op\n",
      "  warnings.warn(\"Initializing zero-element tensors is a no-op\")\n",
      "Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Buffer Warming Up \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mandrekuros\u001b[0m. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n",
      "d:\\Projects\\B-ACE\\b_ace_env_tianshou\\Lib\\site-packages\\wandb\\sdk\\lib\\ipython.py:77: DeprecationWarning: Importing display from IPython.core.display is deprecated since IPython 7.14, please import from IPython display\n",
      "  from IPython.core.display import HTML, display  # type: ignore\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "wandb version 0.16.6 is available!  To upgrade, please run:\n",
       " $ pip install wandb --upgrade"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Tracking run with wandb version 0.16.4"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Run data is saved locally in <code>d:\\Projects\\B-ACE\\B-ACE\\wandb\\run-20240411_132452-PPO_DNN_B_ACE01240411-132423</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Syncing run <strong><a href='https://wandb.ai/andrekuros/B_ACE01/runs/PPO_DNN_B_ACE01240411-132423' target=\"_blank\">PPO_DNN_B_ACE01240411-132423</a></strong> to <a href='https://wandb.ai/andrekuros/B_ACE01' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View project at <a href='https://wandb.ai/andrekuros/B_ACE01' target=\"_blank\">https://wandb.ai/andrekuros/B_ACE01</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       " View run at <a href='https://wandb.ai/andrekuros/B_ACE01/runs/PPO_DNN_B_ACE01240411-132423' target=\"_blank\">https://wandb.ai/andrekuros/B_ACE01/runs/PPO_DNN_B_ACE01240411-132423</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[2], line 216\u001b[0m\n\u001b[0;32m    191\u001b[0m writer\u001b[38;5;241m.\u001b[39mclose()\n\u001b[0;32m    193\u001b[0m \u001b[38;5;66;03m# # ======== Step 5: Run the trainer =========\u001b[39;00m\n\u001b[0;32m    194\u001b[0m \u001b[38;5;66;03m# offPolicyTrainer = OffpolicyTrainer(\u001b[39;00m\n\u001b[0;32m    195\u001b[0m \u001b[38;5;66;03m#     policy=policy,\u001b[39;00m\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    213\u001b[0m            \n\u001b[0;32m    214\u001b[0m \u001b[38;5;66;03m#     )\u001b[39;00m\n\u001b[1;32m--> 216\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[43monPolicyTrainer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    217\u001b[0m writer\u001b[38;5;241m.\u001b[39mclose()\n\u001b[0;32m    218\u001b[0m \u001b[38;5;66;03m# return result, policy.policies[agents[1]]\u001b[39;00m\n",
      "File \u001b[1;32md:\\Projects\\B-ACE\\b_ace_env_tianshou\\Lib\\site-packages\\tianshou\\trainer\\base.py:526\u001b[0m, in \u001b[0;36mBaseTrainer.run\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    524\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m    525\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mis_run \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[1;32m--> 526\u001b[0m     \u001b[43mdeque\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmaxlen\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m)\u001b[49m  \u001b[38;5;66;03m# feed the entire iterator into a zero-length deque\u001b[39;00m\n\u001b[0;32m    527\u001b[0m     info \u001b[38;5;241m=\u001b[39m gather_info(\n\u001b[0;32m    528\u001b[0m         start_time\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstart_time,\n\u001b[0;32m    529\u001b[0m         policy_update_time\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpolicy_update_time,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    534\u001b[0m         test_collector\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtest_collector,\n\u001b[0;32m    535\u001b[0m     )\n\u001b[0;32m    536\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n",
      "File \u001b[1;32md:\\Projects\\B-ACE\\b_ace_env_tianshou\\Lib\\site-packages\\tianshou\\trainer\\base.py:287\u001b[0m, in \u001b[0;36mBaseTrainer.__iter__\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    286\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__iter__\u001b[39m(\u001b[38;5;28mself\u001b[39m):  \u001b[38;5;66;03m# type: ignore\u001b[39;00m\n\u001b[1;32m--> 287\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mreset\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    288\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\n",
      "File \u001b[1;32md:\\Projects\\B-ACE\\b_ace_env_tianshou\\Lib\\site-packages\\tianshou\\trainer\\base.py:263\u001b[0m, in \u001b[0;36mBaseTrainer.reset\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    261\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtest_collector, AsyncCollector)  \u001b[38;5;66;03m# Issue 700\u001b[39;00m\n\u001b[0;32m    262\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtest_collector\u001b[38;5;241m.\u001b[39mreset_stat()\n\u001b[1;32m--> 263\u001b[0m test_result \u001b[38;5;241m=\u001b[39m \u001b[43mtest_episode\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m    264\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpolicy\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    265\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtest_collector\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    266\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtest_fn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    267\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstart_epoch\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    268\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mepisode_per_test\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    269\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlogger\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    270\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43menv_step\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    271\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mreward_metric\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    272\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    273\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m test_result\u001b[38;5;241m.\u001b[39mreturns_stat \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m  \u001b[38;5;66;03m# for mypy\u001b[39;00m\n\u001b[0;32m    274\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbest_epoch \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstart_epoch\n",
      "File \u001b[1;32md:\\Projects\\B-ACE\\b_ace_env_tianshou\\Lib\\site-packages\\tianshou\\trainer\\utils.py:34\u001b[0m, in \u001b[0;36mtest_episode\u001b[1;34m(policy, collector, test_fn, epoch, n_episode, logger, global_step, reward_metric)\u001b[0m\n\u001b[0;32m     32\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m test_fn:\n\u001b[0;32m     33\u001b[0m     test_fn(epoch, global_step)\n\u001b[1;32m---> 34\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[43mcollector\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcollect\u001b[49m\u001b[43m(\u001b[49m\u001b[43mn_episode\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mn_episode\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     35\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m reward_metric:  \u001b[38;5;66;03m# TODO: move into collector\u001b[39;00m\n\u001b[0;32m     36\u001b[0m     rew \u001b[38;5;241m=\u001b[39m reward_metric(result\u001b[38;5;241m.\u001b[39mreturns)\n",
      "File \u001b[1;32md:\\Projects\\B-ACE\\b_ace_env_tianshou\\Lib\\site-packages\\tianshou\\data\\collector.py:317\u001b[0m, in \u001b[0;36mCollector.collect\u001b[1;34m(self, n_step, n_episode, random, render, no_grad, gym_reset_kwargs)\u001b[0m\n\u001b[0;32m    314\u001b[0m action_remap \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpolicy\u001b[38;5;241m.\u001b[39mmap_action(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdata\u001b[38;5;241m.\u001b[39mact)\n\u001b[0;32m    315\u001b[0m \u001b[38;5;66;03m# step in env\u001b[39;00m\n\u001b[1;32m--> 317\u001b[0m obs_next, rew, terminated, truncated, info \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43menv\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstep\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m    318\u001b[0m \u001b[43m    \u001b[49m\u001b[43maction_remap\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    319\u001b[0m \u001b[43m    \u001b[49m\u001b[43mready_env_ids\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    320\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    321\u001b[0m done \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39mlogical_or(terminated, truncated)\n\u001b[0;32m    323\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdata\u001b[38;5;241m.\u001b[39mupdate(\n\u001b[0;32m    324\u001b[0m     obs_next\u001b[38;5;241m=\u001b[39mobs_next,\n\u001b[0;32m    325\u001b[0m     rew\u001b[38;5;241m=\u001b[39mrew,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    329\u001b[0m     info\u001b[38;5;241m=\u001b[39minfo,\n\u001b[0;32m    330\u001b[0m )\n",
      "File \u001b[1;32md:\\Projects\\B-ACE\\b_ace_env_tianshou\\Lib\\site-packages\\tianshou\\env\\venvs.py:276\u001b[0m, in \u001b[0;36mBaseVectorEnv.step\u001b[1;34m(self, action, id)\u001b[0m\n\u001b[0;32m    274\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(action) \u001b[38;5;241m==\u001b[39m \u001b[38;5;28mlen\u001b[39m(\u001b[38;5;28mid\u001b[39m)            \n\u001b[0;32m    275\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m i, j \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28menumerate\u001b[39m(\u001b[38;5;28mid\u001b[39m):\n\u001b[1;32m--> 276\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mworkers\u001b[49m\u001b[43m[\u001b[49m\u001b[43mj\u001b[49m\u001b[43m]\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43maction\u001b[49m\u001b[43m[\u001b[49m\u001b[43mi\u001b[49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    277\u001b[0m result \u001b[38;5;241m=\u001b[39m []\n\u001b[0;32m    278\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m j \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mid\u001b[39m:\n",
      "File \u001b[1;32md:\\Projects\\B-ACE\\b_ace_env_tianshou\\Lib\\site-packages\\tianshou\\env\\worker\\dummy.py:41\u001b[0m, in \u001b[0;36mDummyEnvWorker.send\u001b[1;34m(self, action, **kwargs)\u001b[0m\n\u001b[0;32m     39\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mresult \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39menv\u001b[38;5;241m.\u001b[39mreset(\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m     40\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m---> 41\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mresult \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43menv\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstep\u001b[49m\u001b[43m(\u001b[49m\u001b[43maction\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32md:\\Projects\\B-ACE\\B-ACE\\GodotRLPettingZooWrapper.py:129\u001b[0m, in \u001b[0;36mGodotRLPettingZooWrapper.step\u001b[1;34m(self, actions)\u001b[0m\n\u001b[0;32m    126\u001b[0m     \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mGododtPZWrapper::Error:: Unknow Actions Type -> \u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mactions_type)\n\u001b[0;32m    128\u001b[0m \u001b[38;5;66;03m#print(\"GODOT:\", godot_actions)\u001b[39;00m\n\u001b[1;32m--> 129\u001b[0m obs, reward, dones, truncs, info \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstep\u001b[49m\u001b[43m(\u001b[49m\u001b[43mgodot_actions\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43morder_ij\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n\u001b[0;32m    131\u001b[0m \u001b[38;5;66;03m#print(obs)\u001b[39;00m\n\u001b[0;32m    132\u001b[0m \u001b[38;5;66;03m#self.terminations = []\u001b[39;00m\n\u001b[0;32m    133\u001b[0m \u001b[38;5;66;03m#self.truncations = []\u001b[39;00m\n\u001b[0;32m    134\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mterminations \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n",
      "File \u001b[1;32mD:\\Projects\\B-ACE\\godot_rl_agents\\godot_rl\\core\\godot_env.py:159\u001b[0m, in \u001b[0;36mGodotEnv.step\u001b[1;34m(self, action, order_ij)\u001b[0m\n\u001b[0;32m    148\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m    149\u001b[0m \u001b[38;5;124;03mPerform one step in the environment.\u001b[39;00m\n\u001b[0;32m    150\u001b[0m \n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    156\u001b[0m \u001b[38;5;124;03m    tuple: Tuple containing observation, reward, done flag, termination flag, and info.\u001b[39;00m\n\u001b[0;32m    157\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m    158\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstep_send(action, order_ij\u001b[38;5;241m=\u001b[39morder_ij)\n\u001b[1;32m--> 159\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstep_recv\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32mD:\\Projects\\B-ACE\\godot_rl_agents\\godot_rl\\core\\godot_env.py:183\u001b[0m, in \u001b[0;36mGodotEnv.step_recv\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    176\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mstep_recv\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m    177\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m    178\u001b[0m \u001b[38;5;124;03m    Receive the step response from the Godot environment.\u001b[39;00m\n\u001b[0;32m    179\u001b[0m \n\u001b[0;32m    180\u001b[0m \u001b[38;5;124;03m    Returns:\u001b[39;00m\n\u001b[0;32m    181\u001b[0m \u001b[38;5;124;03m        tuple: Tuple containing observation, reward, done flag, termination flag, and info.\u001b[39;00m\n\u001b[0;32m    182\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[1;32m--> 183\u001b[0m     response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_get_json_dict\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    184\u001b[0m     response[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mobs\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_process_obs(response[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mobs\u001b[39m\u001b[38;5;124m\"\u001b[39m])\n\u001b[0;32m    186\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m (\n\u001b[0;32m    187\u001b[0m         response[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mobs\u001b[39m\u001b[38;5;124m\"\u001b[39m],\n\u001b[0;32m    188\u001b[0m         response[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mreward\u001b[39m\u001b[38;5;124m\"\u001b[39m],\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    191\u001b[0m         [{}] \u001b[38;5;241m*\u001b[39m \u001b[38;5;28mlen\u001b[39m(response[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdone\u001b[39m\u001b[38;5;124m\"\u001b[39m]),\n\u001b[0;32m    192\u001b[0m     )\n",
      "File \u001b[1;32mD:\\Projects\\B-ACE\\godot_rl_agents\\godot_rl\\core\\godot_env.py:371\u001b[0m, in \u001b[0;36mGodotEnv._get_json_dict\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    370\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_get_json_dict\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m--> 371\u001b[0m     data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_get_data\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    372\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m json\u001b[38;5;241m.\u001b[39mloads(data)\n",
      "File \u001b[1;32mD:\\Projects\\B-ACE\\godot_rl_agents\\godot_rl\\core\\godot_env.py:398\u001b[0m, in \u001b[0;36mGodotEnv._get_data\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    395\u001b[0m length: \u001b[38;5;28mint\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m4\u001b[39m\n\u001b[0;32m    397\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m received_length \u001b[38;5;241m<\u001b[39m length:\n\u001b[1;32m--> 398\u001b[0m     data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconnection\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrecv\u001b[49m\u001b[43m(\u001b[49m\u001b[43mlength\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m-\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mreceived_length\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    399\u001b[0m     received_length \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;28mlen\u001b[39m(data)\n\u001b[0;32m    400\u001b[0m     string_size_bytes\u001b[38;5;241m.\u001b[39mextend(data)\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "if __name__ == \"__main__\":\n",
    "                        \n",
    "    torch.set_grad_enabled(True) \n",
    "   \n",
    "    # ======== Step 1: Environment setup =========\n",
    "    train_envs = DummyVectorEnv([_get_env for _ in range(train_env_num)])\n",
    "    test_envs = DummyVectorEnv([_get_env for _ in range(test_env_num)]) \n",
    "\n",
    "    # seed\n",
    "    seed = 100\n",
    "    np.random.seed(seed)\n",
    "    \n",
    "    torch.manual_seed(seed)\n",
    "\n",
    "    #train_envs.seed(seed)\n",
    "    #test_envs.seed(seed)\n",
    "\n",
    "    # ======== Step 2: Agent setup =========\n",
    "    policy, optim, agents = _get_agents()    \n",
    "\n",
    "    \n",
    "    if True:\n",
    "        # ======== Step 3: Collector setup =========\n",
    "        train_collector = Collector(\n",
    "            policy,\n",
    "            train_envs,\n",
    "            #VectorReplayBuffer(300_000, len(train_envs)),\n",
    "            PrioritizedVectorReplayBuffer( 300_000, len(train_envs), alpha=0.6, beta=0.4) , \n",
    "            #ListReplayBuffer(100000)       \n",
    "            # buffer = StateMemoryVectorReplayBuffer(\n",
    "            #         300_000,\n",
    "            #         len(train_envs),  # Assuming train_envs is your vectorized environment\n",
    "            #         memory_size=10,                \n",
    "            #     ),\n",
    "            exploration_noise=True             \n",
    "        )\n",
    "        test_collector = Collector(policy, test_envs, exploration_noise=True)\n",
    "        \n",
    "    else:\n",
    "        agents_buffers_training = {agent : \n",
    "                        PrioritizedVectorReplayBuffer( 300_000, \n",
    "                                                        len(train_envs), \n",
    "                                                        alpha=0.6, \n",
    "                                                        beta=0.4) \n",
    "                                                        for agent in agents\n",
    "                        }\n",
    "        agents_buffers_test = {agent : \n",
    "                        PrioritizedVectorReplayBuffer( 300_000, \n",
    "                                                        len(train_envs), \n",
    "                                                        alpha=0.6, \n",
    "                                                        beta=0.4) \n",
    "                                                        for agent in agents\n",
    "                        }\n",
    "    \n",
    "        # ======== Step 3: Collector setup =========\n",
    "        train_collector = CollectorMA(\n",
    "            policy,\n",
    "            train_envs,\n",
    "            agents_buffers_training, \n",
    "            agents=agents,  # Pass the list of agent IDs                       \n",
    "            exploration_noise=True             \n",
    "        )\n",
    "        test_collector = CollectorMA(policy, test_envs, agents_buffers_test,agents=agents, exploration_noise=True)\n",
    "\n",
    "    \n",
    "        \n",
    "    print(\"Buffer Warming Up \")    \n",
    "    for i in range(trainer_params[\"warmup_size\"]):#int(trainer_params['batch_size'] / (300 * 10 ) )):\n",
    "        \n",
    "         train_collector.collect(n_episode=train_env_num)#,random=True) #trainer_params['batch_size'] * train_env_num))\n",
    "         #train_collector.collect(n_step=300 * 10)\n",
    "         print(\".\", end=\"\") \n",
    "    \n",
    "    # len_buffer = len(train_collector.buffer) / (B_ACE_Config[\"max_cycles\"] * SISL_Config[\"n_pursuers\"])\n",
    "    # print(\"\\nBuffer Lenght: \", len_buffer ) \n",
    "    len_buffer = 0\n",
    "    \n",
    "    info = { \"Buffer\"  : \"PriorizedReplayBuffer\", \" Warmup_ep\" : len_buffer}\n",
    "    \n",
    "    # ======== tensorboard logging setup =========                       \n",
    "    logger = WandbLogger(\n",
    "        train_interval = runConfig[\"EnvConfig\"][\"max_cycles\"] / 400 ,\n",
    "        test_interval = 1,#runConfig[\"max_cycles\"] * runConfig[\"n_pursuers\"],\n",
    "        update_interval = runConfig[\"EnvConfig\"][\"max_cycles\"] / 400,\n",
    "        save_interval = 1,\n",
    "        write_flush = True,\n",
    "        project = \"B_ACE01\",\n",
    "        name = log_name,\n",
    "        entity = None,\n",
    "        run_id = log_name,\n",
    "        config = runConfig,\n",
    "        monitor_gym = True )\n",
    "    \n",
    "    writer = SummaryWriter(log_path)    \n",
    "    writer.add_text(\"args\", str(runConfig))    \n",
    "    logger.load(writer)\n",
    "\n",
    "    global_step_holder = [0] \n",
    "        \n",
    "    # ======== Step 4: Callback functions setup =========\n",
    "    def save_best_fn(policy):                \n",
    "        \n",
    "        if Policy_Config[\"same_policy\"]:\n",
    "            torch.save(policy.policies[agents[0]].state_dict(), model_save_path + \"_\" + str(global_step_holder[0]) + \"_BestRew.pth\")\n",
    "            print(\"Best Saved Rew\" , str(global_step_holder[0]))\n",
    "        \n",
    "        else:\n",
    "            for n,agent in enumerate(agents):\n",
    "                torch.save(policy.policies[agent].state_dict(), model_save_path + \"_\" + str(global_step_holder[0]) + \"_\" + agent + \".pth\")\n",
    "            \n",
    "            print(\"Bests Saved Rew\" , str(global_step_holder[0]))\n",
    "        \n",
    "    def save_test_best_fn(policy):                \n",
    "        \n",
    "        if Policy_Config[\"same_policy\"]:\n",
    "            torch.save(policy.policies[agents[0]].state_dict(), model_save_path + \"_\" + str(global_step_holder[0]) + \"_BestLen.pth\")\n",
    "            print(\"Best Saved Length\" , str(global_step_holder[0]))\n",
    "        \n",
    "        else:\n",
    "            for n,agent in enumerate(agents):\n",
    "                torch.save(policy.policies[agent].state_dict(), model_save_path + \"_\" + str(global_step_holder[0]) + \"_\" + agent + \".pth\")\n",
    "            \n",
    "            print(\"Best Saved Length\" , str(global_step_holder[0]))\n",
    "        \n",
    "\n",
    "    def stop_fn(mean_rewards):\n",
    "        return mean_rewards >= 99999939.0\n",
    "\n",
    "    def train_fn(epoch, env_step):\n",
    "        epsilon = trainer_params['tn_eps_max'] - (trainer_params['tn_eps_max'] - trainer_params['tn_eps_max']/100)*(epoch/trainer_params['max_epoch'])          \n",
    "        if Policy_Config[\"same_policy\"]:\n",
    "            policy.policies[agents[0]].set_eps(epsilon)\n",
    "        else:\n",
    "            for agent in agents:\n",
    "                policy.policies[agent].set_eps(epsilon)\n",
    "                \n",
    "        \n",
    "        # if env_step % 500 == 0:\n",
    "            # logger.write(\"train/env_step\", env_step, {\"train/eps\": eps})\n",
    "\n",
    "\n",
    "    def test_fn(epoch, env_step):\n",
    "               \n",
    "        epsilon = trainer_params['ts_eps_max']#0.01#max(0.001, 0.1 - epoch * 0.001)\n",
    "        if Policy_Config[\"same_policy\"]:\n",
    "            policy.policies[agents[0]].set_eps(epsilon)\n",
    "        else:            \n",
    "            for agent in agents:                             \n",
    "                 policy.policies[agent].set_eps(epsilon)\n",
    "                \n",
    "        \n",
    "        if global_step_holder[0] % 10 == 0:\n",
    "            \n",
    "            if Policy_Config[\"same_policy\"]:\n",
    "                torch.save(policy.policies[agents[0]].state_dict(), model_save_path + \"_\" + str(global_step_holder[0]) + \"_Step.pth\")\n",
    "                print(\"Steps Policy Saved \" , str(global_step_holder[0]))\n",
    "            \n",
    "            else:\n",
    "                for n,agent in enumerate(agents):\n",
    "                    torch.save(policy.policies[agent].state_dict(), model_save_path + \"_\" + str(global_step_holder[0]) + \"_\" + agent + \"Step\" + str(global_step_holder[0]) + \".pth\")\n",
    "                \n",
    "                print(\"Steps Policy Saved \" , str(global_step_holder[0]))\n",
    "\n",
    "        \n",
    "    def reward_metric(rews):       \n",
    "                \n",
    "        global_step_holder[0] +=1 \n",
    "        print(rews)\n",
    "        return np.sum(rews, axis = 1)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "    # # ======== Step 5: Run the trainer =========   \n",
    "    onPolicyTrainer = OnpolicyTrainer(\n",
    "        policy=policy,\n",
    "        train_collector=train_collector,\n",
    "        test_collector=test_collector,\n",
    "        max_epoch=trainer_params['max_epoch'],\n",
    "        step_per_epoch=trainer_params['step_per_epoch'],\n",
    "        repeat_per_collect=20, #TODO: understand\n",
    "        episode_per_test=trainer_params['episode_per_test'],\n",
    "        batch_size=trainer_params['batch_size'],\n",
    "        step_per_collect=trainer_params['step_per_collect'],\n",
    "        stop_fn=stop_fn,\n",
    "        save_best_fn=save_best_fn,\n",
    "        logger=logger,\n",
    "    )\n",
    "    \n",
    "\n",
    "    writer.close()\n",
    "    \n",
    "    # # ======== Step 5: Run the trainer =========\n",
    "    # offPolicyTrainer = OffpolicyTrainer(\n",
    "    #     policy=policy,\n",
    "    #     train_collector=train_collector,\n",
    "    #     test_collector=test_collector,        \n",
    "    #     max_epoch=trainer_params['max_epoch'],\n",
    "    #     step_per_epoch=trainer_params['step_per_epoch'],\n",
    "    #     step_per_collect=trainer_params['step_per_collect'],        \n",
    "    #     episode_per_test= trainer_params['episode_per_test'],\n",
    "    #     batch_size=trainer_params['batch_size'],\n",
    "    #     train_fn=train_fn,\n",
    "    #     test_fn=test_fn,\n",
    "    #     stop_fn=stop_fn,\n",
    "    #     save_best_fn=save_best_fn,\n",
    "    #     # save_test_best_fn=save_test_best_fn,\n",
    "    #     update_per_step=trainer_params['update_per_step'],\n",
    "    #     logger=logger,\n",
    "    #     test_in_train=True,\n",
    "    #     reward_metric=reward_metric,\n",
    "    #     show_progress = True \n",
    "               \n",
    "    #     )\n",
    "    \n",
    "    result = onPolicyTrainer.run()\n",
    "    writer.close()\n",
    "    # return result, policy.policies[agents[1]]\n",
    "    print(f\"\\n==========Result==========\\n{result}\")\n",
    "    print(\"\\n(the trained policy can be accessed via policy.policies[agents[0]])\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}