pip install openrl

conda install -c openrl openrl

# train_ppo.pyfrom openrl.envs.common import makefrom openrl.modules.common import PPONet as Netfrom openrl.runners.common import PPOAgent as Agentenv = make ("CartPole-v1", env_num=9) # 創建環境，并設置環境并行數為 9net = Net (env) # 創建神經網絡agent = Agent (net) # 初始化智能體agent.train (total_time_steps=20000) # 開始訓練，并設置環境運行總步數為 20000

# train_ppo.pyfrom openrl.envs.common import makefrom openrl.modules.common import PPONet as Netfrom openrl.runners.common import PPOAgent as Agentdef train ():# 創建 MPE 環境，使用異步環境，即每個智能體獨立運行env = make ("simple_spread",env_num=100,asynchronous=True,)# 創建 神經網絡，使用 GPU 進行訓練net = Net (env, device="cuda")agent = Agent (net) # 初始化訓練器# 開始訓練agent.train (total_time_steps=5000000)# 保存訓練完成的智能體agent.save ("./ppo_agent/")if __name__ == "__main__":train ()

# mpe_ppo.yamlseed: 0 # 設置 seed，保證每次實驗結果一致lr: 7e-4 # 設置學習率episode_length: 25 # 設置每個 episode 的長度use_recurrent_policy: true # 設置是否使用 RNNuse_joint_action_loss: true # 設置是否使用 JRPO 算法use_valuenorm: true # 設置是否使用 value normalization

python train_ppo.py --config mpe_ppo.yaml

env = make ("simple_spread", env_num=9, render_mode="group_human")

from openrl.envs.wrappers import GIFWrapperenv = GIFWrapper (env, "test_simple_spread.gif")

# test_ppo.pyfrom openrl.envs.common import makefrom openrl.modules.common import PPONet as Netfrom openrl.runners.common import PPOAgent as Agentfrom openrl.envs.wrappers import GIFWrapper # 用于生成 gifdef test ():# 創建 MPE 環境env = make ( "simple_spread", env_num=4)# 使用 GIFWrapper，用于生成 gifenv = GIFWrapper (env, "test_simple_spread.gif")agent = Agent (Net (env)) # 創建 智能體# 保存智能體agent.save ("./ppo_agent/")    # 加載智能體agent.load ("./ppo_agent/")# 開始測試obs, _ = env.reset ()while True:# 智能體根據 observation 預測下一個動作action, _ = agent.act (obs)obs, r, done, info = env.step (action)if done.any ():breakenv.close ()if __name__ == "__main__":test ()

# train_ppo.pyfrom openrl.envs.common import makefrom openrl.modules.common import PPONet as Netfrom openrl.runners.common import PPOAgent as Agentfrom openrl.configs.config import create_config_parserdef train ():# 添加讀取配置文件的代碼cfg_parser = create_config_parser ()cfg = cfg_parser.parse_args ()# 創建 NLP 環境env = make ("daily_dialog",env_num=2,asynchronous=True,cfg=cfg,)net = Net (env, cfg=cfg, device="cuda")agent = Agent (net)agent.train (total_time_steps=5000000)if __name__ == "__main__":train ()

# nlp_ppo.yamldata_path: daily_dialog # 數據集路徑env: # 環境所用到的參數args: {"tokenizer_path": "gpt2"} # 讀取 tokenizer 的路徑seed: 0 # 設置 seed，保證每次實驗結果一致lr: 1e-6 # 設置 policy 模型的學習率critic_lr: 1e-6 # 設置 critic 模型的學習率episode_length: 20 # 設置每個 episode 的長度use_recurrent_policy: true

# nlp_ppo.yaml# 預訓練模型路徑model_path: rajkumarrrk/gpt2-fine-tuned-on-daily-dialog use_share_model: true # 策略網絡和價值網絡是否共享模型ppo_epoch: 5 # ppo 訓練迭代次數data_path: daily_dialog # 數據集名稱或者路徑env: # 環境所用到的參數args: {"tokenizer_path": "gpt2"} # 讀取 tokenizer 的路徑lr: 1e-6 # 設置 policy 模型的學習率critic_lr: 1e-6 # 設置 critic 模型的學習率episode_length: 128 # 設置每個 episode 的長度num_mini_batch: 20

# train_ppo.pyfrom openrl.envs.common import makefrom openrl.modules.common import PPONet as Netfrom openrl.runners.common import PPOAgent as Agentfrom openrl.configs.config import create_config_parserfrom openrl.modules.networks.policy_value_network_gpt import (PolicyValueNetworkGPT as PolicyValueNetwork,)def train ():# 添加讀取配置文件的代碼cfg_parser = create_config_parser ()cfg = cfg_parser.parse_args ()# 創建 NLP 環境env = make ("daily_dialog",env_num=2,asynchronous=True,cfg=cfg,)# 創建自定義神經網絡model_dict = {"model": PolicyValueNetwork}net = Net (env, cfg=cfg, model_dict=model_dict)# 創建訓練智能體agent = Agent (net)agent.train (total_time_steps=5000000)if __name__ == "__main__":train ()

model_dict = {"policy":CustomPolicyNetwork,"critic":CustomValueNetwork,}net = Net (env, model_dict=model_dict)

# nlp_ppo.yamlreward_class:id: NLPReward # 獎勵模型名稱args: {# 用于意圖判斷的模型的名稱或路徑"intent_model": rajkumarrrk/roberta-daily-dialog-intent-classifier,# 用于計算 KL 散度的預訓練模型的名稱或路徑"ref_model": roberta-base, # 用于意圖判斷的 tokenizer 的名稱或路徑}

# train_ppo.pyfromopenrl.rewards.nlp_rewardimportCustomRewardfrom openrl.rewards import RewardFactoryRewardFactory.register("CustomReward",CustomReward)

reward_class:id:"CustomReward"#自定義獎勵模型名稱args: {} # 用戶自定義獎勵函數可能用到的參數

# nlp_ppo.yamlvec_info_class:id: "NLPVecInfo" # 調用 NLPVecInfo 類以打印 NLP 任務中獎勵函數的信息# 設置 wandb 信息wandb_entity: openrl # 這里用于指定 wandb 團隊名稱，請把 openrl 替換為你自己的團隊名稱experiment_name: train_nlp # 這里用于指定實驗名稱run_dir: ./run_results/ # 這里用于指定實驗數據保存的路徑log_interval: 1 # 這里用于指定每隔多少個 episode 上傳一次 wandb 數據# 自行填寫其他參數...

# train_ppo.pyagent.train (total_time_steps=100000, use_wandb=True)

# train_ppo.py # 注冊自定義輸出信息類 VecInfoFactory.register("CustomVecInfo",CustomVecInfo)

# nlp_ppo.yamlvec_info_class:id:"CustomVecInfo"#調用自定義CustomVecInfo類以輸出自定義信息

# nlp_ppo.yamluse_amp: true # 開啟混合精度訓練

# chat.pyfrom openrl.runners.common import ChatAgent as Agentdef chat ():agent = Agent.load ("./ppo_agent", tokenizer="gpt2",)history = []print ("Welcome to OpenRL!")while True:input_text = input ("> User:")if input_text == "quit":breakelif input_text == "reset":history = []print ("Welcome to OpenRL!")continueresponse = agent.chat (input_text, history)print (f"> OpenRL Agent: {response}")history.append (input_text)history.append (response)if __name__ == "__main__":chat ()