From bbc3c3e32dd9bf87188e013268dc983d6ebe22fa Mon Sep 17 00:00:00 2001 From: ChenDRAG <40993476+ChenDRAG@users.noreply.github.com> Date: Thu, 22 Apr 2021 12:49:54 +0800 Subject: [PATCH] Add numerical analysis tool and interactive plot (#341) Co-authored-by: Trinkle23897 --- docs/_static/js/atari | 1 + docs/_static/js/benchmark.js | 67 ++++++++++++++ docs/_static/js/mujoco | 1 + docs/conf.py | 7 ++ docs/tutorials/benchmark.rst | 10 +++ examples/mujoco/README.md | 6 +- examples/mujoco/analysis.py | 89 +++++++++++++++++++ examples/mujoco/benchmark/Ant-v3/result.json | 1 + .../benchmark/HalfCheetah-v3/result.json | 1 + .../mujoco/benchmark/Hopper-v3/result.json | 1 + .../mujoco/benchmark/Humanoid-v3/result.json | 1 + .../InvertedDoublePendulum-v2/result.json | 1 + .../benchmark/InvertedPendulum-v2/result.json | 1 + .../mujoco/benchmark/Reacher-v2/result.json | 1 + .../mujoco/benchmark/Swimmer-v3/result.json | 1 + .../mujoco/benchmark/Walker2d-v3/result.json | 1 + examples/mujoco/gen_json.py | 32 +++++++ examples/mujoco/mujoco_npg.py | 0 examples/mujoco/mujoco_trpo.py | 0 examples/mujoco/plotter.py | 22 +---- examples/mujoco/tools.py | 35 +++++--- 21 files changed, 246 insertions(+), 33 deletions(-) create mode 120000 docs/_static/js/atari create mode 100644 docs/_static/js/benchmark.js create mode 120000 docs/_static/js/mujoco create mode 100755 examples/mujoco/analysis.py create mode 100644 examples/mujoco/benchmark/Ant-v3/result.json create mode 100644 examples/mujoco/benchmark/HalfCheetah-v3/result.json create mode 100644 examples/mujoco/benchmark/Hopper-v3/result.json create mode 100644 examples/mujoco/benchmark/Humanoid-v3/result.json create mode 100644 examples/mujoco/benchmark/InvertedDoublePendulum-v2/result.json create mode 100644 examples/mujoco/benchmark/InvertedPendulum-v2/result.json create mode 100644 examples/mujoco/benchmark/Reacher-v2/result.json create mode 100644 examples/mujoco/benchmark/Swimmer-v3/result.json create mode 100644 examples/mujoco/benchmark/Walker2d-v3/result.json create mode 100755 examples/mujoco/gen_json.py mode change 100644 => 100755 examples/mujoco/mujoco_npg.py mode change 100644 => 100755 examples/mujoco/mujoco_trpo.py diff --git a/docs/_static/js/atari b/docs/_static/js/atari new file mode 120000 index 0000000..6423b4d --- /dev/null +++ b/docs/_static/js/atari @@ -0,0 +1 @@ +../../../examples/atari \ No newline at end of file diff --git a/docs/_static/js/benchmark.js b/docs/_static/js/benchmark.js new file mode 100644 index 0000000..d298101 --- /dev/null +++ b/docs/_static/js/benchmark.js @@ -0,0 +1,67 @@ +var envs = [ + "Ant-v3", + "HalfCheetah-v3", + "Hopper-v3", + "Humanoid-v3", + "InvertedDoublePendulum-v2", + "InvertedPendulum-v2", + "Reacher-v2", + "Swimmer-v3", + "Walker2d-v3", +]; +function showEnv(elem) { + var selectEnv = elem.value || envs[0]; + var dataSource = { + $schema: "https://vega.github.io/schema/vega-lite/v5.json", + data: { + url: "/_static/js/mujoco/benchmark/" + selectEnv + "/result.json" + }, + mark: "line", + height: 400, + width: 800, + params: [{name: "Range", value: 1000000, bind: {input: "range", min: 10000, max: 10000000}}], + transform: [ + {calculate: "datum.rew - datum.rew_std", as: "rew_std0"}, + {calculate: "datum.rew + datum.rew_std", as: "rew_std1"}, + {calculate: "datum.rew + ' ± ' + datum.rew_std", as: "tooltip_str"}, + {filter: "datum.env_step <= Range"}, + ], + encoding: { + color: {"field": "Agent", "type": "nominal"}, + x: {field: "env_step", type: "quantitative", title: "Env step"}, + }, + layer: [{ + "encoding": { + "opacity": {"value": 0.3}, + "y": { + "title": "Return", + "field": "rew_std0", + "type": "quantitative", + }, + "y2": {"field": "rew_std1"}, + tooltip: [ + {field: "env_step", type: "quantitative", title: "Env step"}, + {field: "Agent", type: "nominal"}, + {field: "tooltip_str", type: "nominal", title: "Return"}, + ] + }, + "mark": "area" + }, { + "encoding": { + "y": { + "field": "rew", + "type": "quantitative" + } + }, + "mark": "line" + }] + }; + vegaEmbed("#vis-mujoco", dataSource); +} +$(document).ready(function() { + var envSelect = $("#env-mujoco"); + if (envSelect.length) { + $.each(envs, function(idx, env) {envSelect.append($("").val(env).html(env));}) + showEnv(envSelect); + } +}); diff --git a/docs/_static/js/mujoco b/docs/_static/js/mujoco new file mode 120000 index 0000000..c1c74dc --- /dev/null +++ b/docs/_static/js/mujoco @@ -0,0 +1 @@ +../../../examples/mujoco \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index b981eb4..78f999f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -90,7 +90,14 @@ html_logo = "_static/images/tianshou-logo.png" def setup(app): + app.add_js_file( + "https://cdnjs.cloudflare.com/ajax/libs/jquery/3.6.0/jquery.min.js") + app.add_js_file("https://cdn.jsdelivr.net/npm/vega@5.20.2") + app.add_js_file("https://cdn.jsdelivr.net/npm/vega-lite@5.1.0") + app.add_js_file("https://cdn.jsdelivr.net/npm/vega-embed@6.17.0") + app.add_js_file("js/copybutton.js") + app.add_js_file("js/benchmark.js") app.add_css_file("css/style.css") diff --git a/docs/tutorials/benchmark.rst b/docs/tutorials/benchmark.rst index 8bb74c2..ad8ecee 100644 --- a/docs/tutorials/benchmark.rst +++ b/docs/tutorials/benchmark.rst @@ -8,6 +8,16 @@ Tianshou's Mujoco benchmark contains state-of-the-art results (even better than Please refer to https://github.com/thu-ml/tianshou/tree/master/examples/mujoco +.. raw:: html + +
+ +
+
+
+
+ + Atari Benchmark --------------- diff --git a/examples/mujoco/README.md b/examples/mujoco/README.md index bad3176..ff37db4 100644 --- a/examples/mujoco/README.md +++ b/examples/mujoco/README.md @@ -16,7 +16,7 @@ Supported algorithms are listed below: - [Twin Delayed DDPG (TD3)](https://arxiv.org/pdf/1802.09477.pdf), [commit id](https://github.com/thu-ml/tianshou/tree/e605bdea942b408126ef4fbc740359773259c9ec) - [Soft Actor-Critic (SAC)](https://arxiv.org/pdf/1812.05905.pdf), [commit id](https://github.com/thu-ml/tianshou/tree/e605bdea942b408126ef4fbc740359773259c9ec) - [REINFORCE algorithm](https://papers.nips.cc/paper/1999/file/464d828b85b0bed98e80ade0a5c43b0f-Paper.pdf), [commit id](https://github.com/thu-ml/tianshou/tree/e27b5a26f330de446fe15388bf81c3777f024fb9) -- [Natural Policy Gradient](https://proceedings.neurips.cc/paper/2001/file/4b86abe48d358ecf194c56c69108433e-Paper.pdf), [commit id](https://github.com/thu-ml/tianshou/tree/1dcf65fe21dc7636966796b6099ede1f4bd775e1) +- [Natural Policy Gradient](https://proceedings.neurips.cc/paper/2001/file/4b86abe48d358ecf194c56c69108433e-Paper.pdf), [commit id](https://github.com/thu-ml/tianshou/tree/844d7703c313009c4c364edb4018c91de93439ca) - [Advantage Actor-Critic (A2C)](https://openai.com/blog/baselines-acktr-a2c/), [commit id](https://github.com/thu-ml/tianshou/tree/1730a9008ad6bb67cac3b21347bed33b532b17bc) - [Proximal Policy Optimization (PPO)](https://arxiv.org/pdf/1707.06347.pdf), [commit id](https://github.com/thu-ml/tianshou/tree/6426a39796db052bafb7cabe85c764db20a722b0) - [Trust Region Policy Optimization (TRPO)](https://arxiv.org/pdf/1502.05477.pdf), [commit id](https://github.com/thu-ml/tianshou/tree/5057b5c89e6168220272c9c28a15b758a72efc32) @@ -46,8 +46,12 @@ This will start 10 experiments with different seeds. Now that all the experiments are finished, we can convert all tfevent files into csv files and then try plotting the results. ```bash +# geenrate csv $ ./tools.py --root-dir ./results/Ant-v3/sac +# generate figures $ ./plotter.py --root-dir ./results/Ant-v3 --shaded-std --legend-pattern "\\w+" +# generate numerical result (support multiple groups: `--root-dir ./` instead of single dir) +$ ./analysis.py --root-dir ./results --norm ``` #### Example benchmark diff --git a/examples/mujoco/analysis.py b/examples/mujoco/analysis.py new file mode 100755 index 0000000..01a2cf6 --- /dev/null +++ b/examples/mujoco/analysis.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 + +import re +import argparse +import numpy as np +from tabulate import tabulate +from collections import defaultdict +from tools import find_all_files, group_files, csv2numpy + + +def numerical_anysis(root_dir, xlim, norm=False): + file_pattern = re.compile(r".*/test_rew_\d+seeds.csv$") + norm_group_pattern = re.compile(r"(/|^)\w+?\-v(\d|$)") + output_group_pattern = re.compile(r".*?(?=(/|^)\w+?\-v\d)") + csv_files = find_all_files(root_dir, file_pattern) + norm_group = group_files(csv_files, norm_group_pattern) + output_group = group_files(csv_files, output_group_pattern) + # calculate numerical outcome for each csv_file (y/std integration max_y, final_y) + results = defaultdict(list) + for f in csv_files: + result = csv2numpy(f) + if norm: + result = np.stack([ + result['env_step'], + result['rew'] - result['rew'][0], + result['rew:shaded']]) + else: + result = np.stack([ + result['env_step'], result['rew'], result['rew:shaded']]) + + if result[0, -1] < xlim: + continue + + final_rew = np.interp(xlim, result[0], result[1]) + final_rew_std = np.interp(xlim, result[0], result[2]) + result = result[:, result[0] <= xlim] + + if len(result) == 0: + continue + + if result[0, -1] < xlim: + last_line = np.array([xlim, final_rew, final_rew_std]).reshape(3, 1) + result = np.concatenate([result, last_line], axis=-1) + + max_id = np.argmax(result[1]) + results['name'].append(f) + results['final_reward'].append(result[1, -1]) + results['final_reward_std'].append(result[2, -1]) + results['max_reward'].append(result[1, max_id]) + results['max_std'].append(result[2, max_id]) + results['reward_integration'].append(np.trapz(result[1], x=result[0])) + results['reward_std_integration'].append(np.trapz(result[2], x=result[0])) + + results = {k: np.array(v) for k, v in results.items()} + print(tabulate(results, headers="keys")) + + if norm: + # calculate normalized numerical outcome for each csv_file group + for _, fs in norm_group.items(): + mask = np.isin(results['name'], fs) + for k, v in results.items(): + if k == 'name': + continue + v[mask] = v[mask] / max(v[mask]) + # Add all numerical results for each outcome group + group_results = defaultdict(list) + for g, fs in output_group.items(): + group_results['name'].append(g) + mask = np.isin(results['name'], fs) + group_results['num'].append(sum(mask)) + for k in results.keys(): + if k == 'name': + continue + group_results[k + ":norm"].append(results[k][mask].mean()) + # print all outputs for each csv_file and each outcome group + print() + print(tabulate(group_results, headers="keys")) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--xlim', type=int, default=1000000, + help='x-axis limitation (default: 1000000)') + parser.add_argument('--root-dir', type=str) + parser.add_argument( + '--norm', action="store_true", + help="Normalize all results according to environment.") + args = parser.parse_args() + numerical_anysis(args.root_dir, args.xlim, norm=args.norm) diff --git a/examples/mujoco/benchmark/Ant-v3/result.json b/examples/mujoco/benchmark/Ant-v3/result.json new file mode 100644 index 0000000..8716d53 --- /dev/null +++ b/examples/mujoco/benchmark/Ant-v3/result.json @@ -0,0 +1 @@ +[{"env_step": 0, "rew": 990.3785, "rew_std": 4.297, "Agent": "SAC"}, {"env_step": 5000, "rew": 634.386, "rew_std": 184.9621, "Agent": "SAC"}, {"env_step": 10000, "rew": 466.3264, "rew_std": 104.4639, "Agent": "SAC"}, {"env_step": 15000, "rew": 472.4038, "rew_std": 142.9891, "Agent": "SAC"}, {"env_step": 20000, "rew": 506.0177, "rew_std": 161.9189, "Agent": "SAC"}, {"env_step": 25000, "rew": 603.8001, "rew_std": 120.5751, "Agent": "SAC"}, {"env_step": 30000, "rew": 608.1596, "rew_std": 147.4683, "Agent": "SAC"}, {"env_step": 35000, "rew": 596.8617, "rew_std": 185.3435, "Agent": "SAC"}, {"env_step": 40000, "rew": 605.0013, "rew_std": 123.9596, "Agent": "SAC"}, {"env_step": 45000, "rew": 682.4043, "rew_std": 126.4225, "Agent": "SAC"}, {"env_step": 50000, "rew": 711.1211, "rew_std": 181.235, "Agent": "SAC"}, {"env_step": 55000, "rew": 712.1348, "rew_std": 158.3893, "Agent": "SAC"}, {"env_step": 60000, "rew": 668.6808, "rew_std": 186.2096, "Agent": "SAC"}, {"env_step": 65000, "rew": 803.7135, "rew_std": 128.7597, "Agent": "SAC"}, {"env_step": 70000, "rew": 836.9211, "rew_std": 249.2294, "Agent": "SAC"}, {"env_step": 75000, "rew": 812.1804, "rew_std": 203.7858, "Agent": "SAC"}, {"env_step": 80000, "rew": 816.8486, "rew_std": 280.3634, "Agent": "SAC"}, {"env_step": 85000, "rew": 838.3772, "rew_std": 331.4502, "Agent": "SAC"}, {"env_step": 90000, "rew": 901.242, "rew_std": 295.797, "Agent": "SAC"}, {"env_step": 95000, "rew": 859.4468, "rew_std": 261.5593, "Agent": "SAC"}, {"env_step": 100000, "rew": 1019.8017, "rew_std": 449.9045, "Agent": "SAC"}, {"env_step": 105000, "rew": 1030.3104, "rew_std": 286.2318, "Agent": "SAC"}, {"env_step": 110000, "rew": 963.0668, "rew_std": 311.5471, "Agent": "SAC"}, {"env_step": 115000, "rew": 1089.9735, "rew_std": 519.6678, "Agent": "SAC"}, {"env_step": 120000, "rew": 1178.9178, "rew_std": 389.9193, "Agent": "SAC"}, {"env_step": 125000, "rew": 1322.1717, "rew_std": 393.7724, "Agent": "SAC"}, {"env_step": 130000, "rew": 1312.6976, "rew_std": 469.6964, "Agent": "SAC"}, {"env_step": 135000, "rew": 1347.1568, "rew_std": 414.4463, "Agent": "SAC"}, {"env_step": 140000, "rew": 1386.2821, "rew_std": 553.034, "Agent": "SAC"}, {"env_step": 145000, "rew": 1585.8144, "rew_std": 678.5971, "Agent": "SAC"}, {"env_step": 150000, "rew": 1426.0943, "rew_std": 782.1992, "Agent": "SAC"}, {"env_step": 155000, "rew": 1485.7579, "rew_std": 816.0993, "Agent": "SAC"}, {"env_step": 160000, "rew": 1335.9676, "rew_std": 526.1357, "Agent": "SAC"}, {"env_step": 165000, "rew": 1727.0195, "rew_std": 949.2217, "Agent": "SAC"}, {"env_step": 170000, "rew": 1815.7559, "rew_std": 780.9618, "Agent": "SAC"}, {"env_step": 175000, "rew": 1679.7342, "rew_std": 608.2833, "Agent": "SAC"}, {"env_step": 180000, "rew": 1755.6202, "rew_std": 908.6728, "Agent": "SAC"}, {"env_step": 185000, "rew": 2060.7492, "rew_std": 798.1616, "Agent": "SAC"}, {"env_step": 190000, "rew": 2071.3926, "rew_std": 891.6286, "Agent": "SAC"}, {"env_step": 195000, "rew": 1972.7574, "rew_std": 905.4189, "Agent": "SAC"}, {"env_step": 200000, "rew": 2132.5654, "rew_std": 977.1712, "Agent": "SAC"}, {"env_step": 205000, "rew": 2025.7295, "rew_std": 691.8455, "Agent": "SAC"}, {"env_step": 210000, "rew": 2247.2497, "rew_std": 695.7745, "Agent": "SAC"}, {"env_step": 215000, "rew": 2227.0186, "rew_std": 914.713, "Agent": "SAC"}, {"env_step": 220000, "rew": 2743.7285, "rew_std": 978.705, "Agent": "SAC"}, {"env_step": 225000, "rew": 2755.1179, "rew_std": 976.5856, "Agent": "SAC"}, {"env_step": 230000, "rew": 2240.208, "rew_std": 758.3069, "Agent": "SAC"}, {"env_step": 235000, "rew": 2476.1974, "rew_std": 1088.2916, "Agent": "SAC"}, {"env_step": 240000, "rew": 2759.6599, "rew_std": 868.8349, "Agent": "SAC"}, {"env_step": 245000, "rew": 2654.4313, "rew_std": 1008.0752, "Agent": "SAC"}, {"env_step": 250000, "rew": 2892.8498, "rew_std": 1175.0901, "Agent": "SAC"}, {"env_step": 255000, "rew": 3023.2165, "rew_std": 936.6243, "Agent": "SAC"}, {"env_step": 260000, "rew": 2987.143, "rew_std": 954.9922, "Agent": "SAC"}, {"env_step": 265000, "rew": 3086.6823, "rew_std": 878.5816, "Agent": "SAC"}, {"env_step": 270000, "rew": 2476.7571, "rew_std": 831.4616, "Agent": "SAC"}, {"env_step": 275000, "rew": 3028.8691, "rew_std": 810.8356, "Agent": "SAC"}, {"env_step": 280000, "rew": 3236.7877, "rew_std": 1200.7687, "Agent": "SAC"}, {"env_step": 285000, "rew": 3482.0228, "rew_std": 1098.2739, "Agent": "SAC"}, {"env_step": 290000, "rew": 3599.3998, "rew_std": 707.1372, "Agent": "SAC"}, {"env_step": 295000, "rew": 3509.0719, "rew_std": 898.0675, "Agent": "SAC"}, {"env_step": 300000, "rew": 3560.7321, "rew_std": 779.5421, "Agent": "SAC"}, {"env_step": 305000, "rew": 3496.5751, "rew_std": 1100.757, "Agent": "SAC"}, {"env_step": 310000, "rew": 3608.0523, "rew_std": 971.3736, "Agent": "SAC"}, {"env_step": 315000, "rew": 3281.9811, "rew_std": 1226.0263, "Agent": "SAC"}, {"env_step": 320000, "rew": 3766.0278, "rew_std": 1120.3099, "Agent": "SAC"}, {"env_step": 325000, "rew": 4018.244, "rew_std": 799.463, "Agent": "SAC"}, {"env_step": 330000, "rew": 3653.1125, "rew_std": 1122.2692, "Agent": "SAC"}, {"env_step": 335000, "rew": 3968.1996, "rew_std": 848.7675, "Agent": "SAC"}, {"env_step": 340000, "rew": 3903.1583, "rew_std": 586.599, "Agent": "SAC"}, {"env_step": 345000, "rew": 3799.5278, "rew_std": 897.6643, "Agent": "SAC"}, {"env_step": 350000, "rew": 3659.2519, "rew_std": 813.1146, "Agent": "SAC"}, {"env_step": 355000, "rew": 4339.5464, "rew_std": 627.7838, "Agent": "SAC"}, {"env_step": 360000, "rew": 3534.6213, "rew_std": 978.4176, "Agent": "SAC"}, {"env_step": 365000, "rew": 3823.3135, "rew_std": 840.9491, "Agent": "SAC"}, {"env_step": 370000, "rew": 4312.3502, "rew_std": 707.5655, "Agent": "SAC"}, {"env_step": 375000, "rew": 3882.813, "rew_std": 899.9477, "Agent": "SAC"}, {"env_step": 380000, "rew": 4151.4361, "rew_std": 886.019, "Agent": "SAC"}, {"env_step": 385000, "rew": 4247.4278, "rew_std": 817.3344, "Agent": "SAC"}, {"env_step": 390000, "rew": 4193.3913, "rew_std": 1289.8256, "Agent": "SAC"}, {"env_step": 395000, "rew": 4262.8331, "rew_std": 856.4907, "Agent": "SAC"}, {"env_step": 400000, "rew": 4417.5728, "rew_std": 710.4488, "Agent": "SAC"}, {"env_step": 405000, "rew": 4215.4565, "rew_std": 737.7075, "Agent": "SAC"}, {"env_step": 410000, "rew": 4656.0872, "rew_std": 618.7525, "Agent": "SAC"}, {"env_step": 415000, "rew": 4324.2753, "rew_std": 1087.3486, "Agent": "SAC"}, {"env_step": 420000, "rew": 4373.4924, "rew_std": 1044.2813, "Agent": "SAC"}, {"env_step": 425000, "rew": 4343.1537, "rew_std": 650.1327, "Agent": "SAC"}, {"env_step": 430000, "rew": 4477.3905, "rew_std": 881.3755, "Agent": "SAC"}, {"env_step": 435000, "rew": 4355.3773, "rew_std": 944.8012, "Agent": "SAC"}, {"env_step": 440000, "rew": 4675.7451, "rew_std": 647.9754, "Agent": "SAC"}, {"env_step": 445000, "rew": 4120.4879, "rew_std": 1154.4533, "Agent": "SAC"}, {"env_step": 450000, "rew": 4728.8119, "rew_std": 754.292, "Agent": "SAC"}, {"env_step": 455000, "rew": 4705.7272, "rew_std": 691.9497, "Agent": "SAC"}, {"env_step": 460000, "rew": 4537.7941, "rew_std": 1034.1699, "Agent": "SAC"}, {"env_step": 465000, "rew": 4082.6977, "rew_std": 1489.3087, "Agent": "SAC"}, {"env_step": 470000, "rew": 4350.9532, "rew_std": 1500.3143, "Agent": "SAC"}, {"env_step": 475000, "rew": 4122.221, "rew_std": 1546.766, "Agent": "SAC"}, {"env_step": 480000, "rew": 4725.6855, "rew_std": 949.7799, "Agent": "SAC"}, {"env_step": 485000, "rew": 4804.9972, "rew_std": 707.0428, "Agent": "SAC"}, {"env_step": 490000, "rew": 4877.2005, "rew_std": 643.9763, "Agent": "SAC"}, {"env_step": 495000, "rew": 5034.0545, "rew_std": 475.6285, "Agent": "SAC"}, {"env_step": 500000, "rew": 5128.7439, "rew_std": 518.8488, "Agent": "SAC"}, {"env_step": 505000, "rew": 5069.8386, "rew_std": 636.3886, "Agent": "SAC"}, {"env_step": 510000, "rew": 5159.2177, "rew_std": 596.1149, "Agent": "SAC"}, {"env_step": 515000, "rew": 4712.5957, "rew_std": 827.3416, "Agent": "SAC"}, {"env_step": 520000, "rew": 4981.1566, "rew_std": 963.9806, "Agent": "SAC"}, {"env_step": 525000, "rew": 4913.5781, "rew_std": 737.1729, "Agent": "SAC"}, {"env_step": 530000, "rew": 4850.7048, "rew_std": 852.6721, "Agent": "SAC"}, {"env_step": 535000, "rew": 4963.8677, "rew_std": 858.5452, "Agent": "SAC"}, {"env_step": 540000, "rew": 5204.4067, "rew_std": 472.5198, "Agent": "SAC"}, {"env_step": 545000, "rew": 5012.2747, "rew_std": 705.919, "Agent": "SAC"}, {"env_step": 550000, "rew": 5167.2548, "rew_std": 609.7783, "Agent": "SAC"}, {"env_step": 555000, "rew": 4387.9413, "rew_std": 2043.9651, "Agent": "SAC"}, {"env_step": 560000, "rew": 4758.9642, "rew_std": 1701.0235, "Agent": "SAC"}, {"env_step": 565000, "rew": 4160.9355, "rew_std": 2370.4786, "Agent": "SAC"}, {"env_step": 570000, "rew": 4081.5274, "rew_std": 2229.5623, "Agent": "SAC"}, {"env_step": 575000, "rew": 4145.1935, "rew_std": 2112.6704, "Agent": "SAC"}, {"env_step": 580000, "rew": 4558.6605, "rew_std": 1902.1764, "Agent": "SAC"}, {"env_step": 585000, "rew": 4824.5614, "rew_std": 1853.2917, "Agent": "SAC"}, {"env_step": 590000, "rew": 4470.5525, "rew_std": 1523.8954, "Agent": "SAC"}, {"env_step": 595000, "rew": 5261.3918, "rew_std": 814.7524, "Agent": "SAC"}, {"env_step": 600000, "rew": 4924.1725, "rew_std": 720.2538, "Agent": "SAC"}, {"env_step": 605000, "rew": 5051.8181, "rew_std": 548.6794, "Agent": "SAC"}, {"env_step": 610000, "rew": 4967.6005, "rew_std": 925.7063, "Agent": "SAC"}, {"env_step": 615000, "rew": 5394.8005, "rew_std": 639.5644, "Agent": "SAC"}, {"env_step": 620000, "rew": 5062.7683, "rew_std": 724.4078, "Agent": "SAC"}, {"env_step": 625000, "rew": 5078.1642, "rew_std": 702.1159, "Agent": "SAC"}, {"env_step": 630000, "rew": 5317.6296, "rew_std": 437.7749, "Agent": "SAC"}, {"env_step": 635000, "rew": 4704.1758, "rew_std": 2060.9637, "Agent": "SAC"}, {"env_step": 640000, "rew": 4843.6902, "rew_std": 1808.5671, "Agent": "SAC"}, {"env_step": 645000, "rew": 4581.3947, "rew_std": 1571.9396, "Agent": "SAC"}, {"env_step": 650000, "rew": 4826.9942, "rew_std": 1221.073, "Agent": "SAC"}, {"env_step": 655000, "rew": 5288.6551, "rew_std": 490.8404, "Agent": "SAC"}, {"env_step": 660000, "rew": 5335.5634, "rew_std": 508.8307, "Agent": "SAC"}, {"env_step": 665000, "rew": 5266.9036, "rew_std": 517.7074, "Agent": "SAC"}, {"env_step": 670000, "rew": 5547.8741, "rew_std": 354.7182, "Agent": "SAC"}, {"env_step": 675000, "rew": 5328.8314, "rew_std": 489.3294, "Agent": "SAC"}, {"env_step": 680000, "rew": 5405.7608, "rew_std": 288.6692, "Agent": "SAC"}, {"env_step": 685000, "rew": 5411.7402, "rew_std": 503.3251, "Agent": "SAC"}, {"env_step": 690000, "rew": 5260.8163, "rew_std": 502.3217, "Agent": "SAC"}, {"env_step": 695000, "rew": 5450.3069, "rew_std": 519.2909, "Agent": "SAC"}, {"env_step": 700000, "rew": 5336.7194, "rew_std": 764.3954, "Agent": "SAC"}, {"env_step": 705000, "rew": 5491.0999, "rew_std": 602.9351, "Agent": "SAC"}, {"env_step": 710000, "rew": 5338.5924, "rew_std": 430.7632, "Agent": "SAC"}, {"env_step": 715000, "rew": 5694.361, "rew_std": 227.2388, "Agent": "SAC"}, {"env_step": 720000, "rew": 5406.0092, "rew_std": 443.1683, "Agent": "SAC"}, {"env_step": 725000, "rew": 5575.3379, "rew_std": 379.3146, "Agent": "SAC"}, {"env_step": 730000, "rew": 5475.6525, "rew_std": 482.8181, "Agent": "SAC"}, {"env_step": 735000, "rew": 5315.6012, "rew_std": 605.0353, "Agent": "SAC"}, {"env_step": 740000, "rew": 5569.833, "rew_std": 348.5342, "Agent": "SAC"}, {"env_step": 745000, "rew": 5545.2522, "rew_std": 556.2845, "Agent": "SAC"}, {"env_step": 750000, "rew": 5725.0663, "rew_std": 311.8731, "Agent": "SAC"}, {"env_step": 755000, "rew": 5690.6184, "rew_std": 351.7627, "Agent": "SAC"}, {"env_step": 760000, "rew": 5567.3753, "rew_std": 482.2716, "Agent": "SAC"}, {"env_step": 765000, "rew": 5510.4696, "rew_std": 385.251, "Agent": "SAC"}, {"env_step": 770000, "rew": 5270.3259, "rew_std": 987.3123, "Agent": "SAC"}, {"env_step": 775000, "rew": 5467.4195, "rew_std": 853.2007, "Agent": "SAC"}, {"env_step": 780000, "rew": 5287.154, "rew_std": 1035.4332, "Agent": "SAC"}, {"env_step": 785000, "rew": 5691.6446, "rew_std": 516.7474, "Agent": "SAC"}, {"env_step": 790000, "rew": 5591.892, "rew_std": 295.6546, "Agent": "SAC"}, {"env_step": 795000, "rew": 5488.8903, "rew_std": 424.9128, "Agent": "SAC"}, {"env_step": 800000, "rew": 5459.0244, "rew_std": 826.6396, "Agent": "SAC"}, {"env_step": 805000, "rew": 5664.4516, "rew_std": 397.5254, "Agent": "SAC"}, {"env_step": 810000, "rew": 5543.2376, "rew_std": 562.8029, "Agent": "SAC"}, {"env_step": 815000, "rew": 5693.9674, "rew_std": 408.3632, "Agent": "SAC"}, {"env_step": 820000, "rew": 5680.3464, "rew_std": 307.6037, "Agent": "SAC"}, {"env_step": 825000, "rew": 5709.4142, "rew_std": 227.2665, "Agent": "SAC"}, {"env_step": 830000, "rew": 5555.0137, "rew_std": 505.6524, "Agent": "SAC"}, {"env_step": 835000, "rew": 5673.183, "rew_std": 379.6022, "Agent": "SAC"}, {"env_step": 840000, "rew": 5107.2045, "rew_std": 1424.1345, "Agent": "SAC"}, {"env_step": 845000, "rew": 5517.1734, "rew_std": 693.1669, "Agent": "SAC"}, {"env_step": 850000, "rew": 5629.4855, "rew_std": 347.9435, "Agent": "SAC"}, {"env_step": 855000, "rew": 5564.5545, "rew_std": 534.6357, "Agent": "SAC"}, {"env_step": 860000, "rew": 5395.1586, "rew_std": 782.8633, "Agent": "SAC"}, {"env_step": 865000, "rew": 5656.6156, "rew_std": 360.7084, "Agent": "SAC"}, {"env_step": 870000, "rew": 5845.8685, "rew_std": 260.3917, "Agent": "SAC"}, {"env_step": 875000, "rew": 5446.3008, "rew_std": 735.6111, "Agent": "SAC"}, {"env_step": 880000, "rew": 5640.0635, "rew_std": 470.5824, "Agent": "SAC"}, {"env_step": 885000, "rew": 5732.5999, "rew_std": 502.8969, "Agent": "SAC"}, {"env_step": 890000, "rew": 5555.4769, "rew_std": 465.2802, "Agent": "SAC"}, {"env_step": 895000, "rew": 5651.3705, "rew_std": 230.4769, "Agent": "SAC"}, {"env_step": 900000, "rew": 5702.166, "rew_std": 372.4946, "Agent": "SAC"}, {"env_step": 905000, "rew": 5720.2143, "rew_std": 355.2288, "Agent": "SAC"}, {"env_step": 910000, "rew": 5585.9028, "rew_std": 563.3985, "Agent": "SAC"}, {"env_step": 915000, "rew": 5672.2431, "rew_std": 522.0986, "Agent": "SAC"}, {"env_step": 920000, "rew": 5778.5705, "rew_std": 412.6784, "Agent": "SAC"}, {"env_step": 925000, "rew": 5810.2684, "rew_std": 324.2981, "Agent": "SAC"}, {"env_step": 930000, "rew": 5125.3163, "rew_std": 1923.0686, "Agent": "SAC"}, {"env_step": 935000, "rew": 4851.2896, "rew_std": 2254.6923, "Agent": "SAC"}, {"env_step": 940000, "rew": 5244.8474, "rew_std": 1666.2577, "Agent": "SAC"}, {"env_step": 945000, "rew": 5303.0647, "rew_std": 1354.3917, "Agent": "SAC"}, {"env_step": 950000, "rew": 5653.4382, "rew_std": 624.9922, "Agent": "SAC"}, {"env_step": 955000, "rew": 5778.0612, "rew_std": 426.7755, "Agent": "SAC"}, {"env_step": 960000, "rew": 5670.2887, "rew_std": 693.5821, "Agent": "SAC"}, {"env_step": 965000, "rew": 5044.8635, "rew_std": 1861.1285, "Agent": "SAC"}, {"env_step": 970000, "rew": 4849.5535, "rew_std": 2068.7704, "Agent": "SAC"}, {"env_step": 975000, "rew": 5113.4048, "rew_std": 1735.4742, "Agent": "SAC"}, {"env_step": 980000, "rew": 5188.3984, "rew_std": 1727.6241, "Agent": "SAC"}, {"env_step": 985000, "rew": 5850.2488, "rew_std": 475.6532, "Agent": "SAC"}, {"env_step": 990000, "rew": 5813.4523, "rew_std": 264.809, "Agent": "SAC"}, {"env_step": 995000, "rew": 5742.2444, "rew_std": 418.5699, "Agent": "SAC"}, {"env_step": 1000000, "rew": 5714.2661, "rew_std": 368.5243, "Agent": "SAC"}, {"env_step": 1005000, "rew": 5413.1424, "rew_std": 797.9577, "Agent": "SAC"}, {"env_step": 1010000, "rew": 5183.4618, "rew_std": 2031.9834, "Agent": "SAC"}, {"env_step": 1015000, "rew": 5119.8702, "rew_std": 1840.3876, "Agent": "SAC"}, {"env_step": 1020000, "rew": 5168.524, "rew_std": 1586.408, "Agent": "SAC"}, {"env_step": 1025000, "rew": 5733.1041, "rew_std": 317.4068, "Agent": "SAC"}, {"env_step": 1030000, "rew": 5642.594, "rew_std": 248.2455, "Agent": "SAC"}, {"env_step": 1035000, "rew": 5850.1509, "rew_std": 411.4642, "Agent": "SAC"}, {"env_step": 1040000, "rew": 5974.4491, "rew_std": 269.7785, "Agent": "SAC"}, {"env_step": 1045000, "rew": 5031.7685, "rew_std": 2330.7907, "Agent": "SAC"}, {"env_step": 1050000, "rew": 5236.7993, "rew_std": 1973.0137, "Agent": "SAC"}, {"env_step": 1055000, "rew": 5021.8125, "rew_std": 2004.5906, "Agent": "SAC"}, {"env_step": 1060000, "rew": 5165.6284, "rew_std": 2382.8055, "Agent": "SAC"}, {"env_step": 1065000, "rew": 5158.7046, "rew_std": 1851.172, "Agent": "SAC"}, {"env_step": 1070000, "rew": 5138.9775, "rew_std": 2083.5424, "Agent": "SAC"}, {"env_step": 1075000, "rew": 5019.4091, "rew_std": 2127.5281, "Agent": "SAC"}, {"env_step": 1080000, "rew": 5292.9344, "rew_std": 2149.235, "Agent": "SAC"}, {"env_step": 1085000, "rew": 5154.5682, "rew_std": 1838.3159, "Agent": "SAC"}, {"env_step": 1090000, "rew": 5282.5727, "rew_std": 1822.7381, "Agent": "SAC"}, {"env_step": 1095000, "rew": 4668.0528, "rew_std": 2605.2238, "Agent": "SAC"}, {"env_step": 1100000, "rew": 4801.5646, "rew_std": 2266.8663, "Agent": "SAC"}, {"env_step": 1105000, "rew": 4975.1772, "rew_std": 2411.4807, "Agent": "SAC"}, {"env_step": 1110000, "rew": 5378.207, "rew_std": 1623.0413, "Agent": "SAC"}, {"env_step": 1115000, "rew": 5516.254, "rew_std": 841.5805, "Agent": "SAC"}, {"env_step": 1120000, "rew": 5419.4915, "rew_std": 888.8777, "Agent": "SAC"}, {"env_step": 1125000, "rew": 5582.7173, "rew_std": 1128.1217, "Agent": "SAC"}, {"env_step": 1130000, "rew": 5497.8511, "rew_std": 1006.239, "Agent": "SAC"}, {"env_step": 1135000, "rew": 5839.487, "rew_std": 431.9108, "Agent": "SAC"}, {"env_step": 1140000, "rew": 5961.432, "rew_std": 378.7637, "Agent": "SAC"}, {"env_step": 1145000, "rew": 5857.0817, "rew_std": 374.6548, "Agent": "SAC"}, {"env_step": 1150000, "rew": 5925.9246, "rew_std": 342.4812, "Agent": "SAC"}, {"env_step": 1155000, "rew": 5907.3666, "rew_std": 498.7652, "Agent": "SAC"}, {"env_step": 1160000, "rew": 6098.61, "rew_std": 263.329, "Agent": "SAC"}, {"env_step": 1165000, "rew": 5864.8649, "rew_std": 387.2574, "Agent": "SAC"}, {"env_step": 1170000, "rew": 5890.1991, "rew_std": 508.4673, "Agent": "SAC"}, {"env_step": 1175000, "rew": 5840.934, "rew_std": 384.9363, "Agent": "SAC"}, {"env_step": 1180000, "rew": 5641.1752, "rew_std": 507.7719, "Agent": "SAC"}, {"env_step": 1185000, "rew": 5885.6678, "rew_std": 434.8161, "Agent": "SAC"}, {"env_step": 1190000, "rew": 5746.5579, "rew_std": 341.3485, "Agent": "SAC"}, {"env_step": 1195000, "rew": 5856.6325, "rew_std": 438.0519, "Agent": "SAC"}, {"env_step": 1200000, "rew": 5912.71, "rew_std": 225.9145, "Agent": "SAC"}, {"env_step": 1205000, "rew": 6033.2897, "rew_std": 362.7604, "Agent": "SAC"}, {"env_step": 1210000, "rew": 5916.6141, "rew_std": 264.2832, "Agent": "SAC"}, {"env_step": 1215000, "rew": 6096.7137, "rew_std": 276.9315, "Agent": "SAC"}, {"env_step": 1220000, "rew": 6058.312, "rew_std": 220.9652, "Agent": "SAC"}, {"env_step": 1225000, "rew": 6058.1806, "rew_std": 321.2416, "Agent": "SAC"}, {"env_step": 1230000, "rew": 5876.7533, "rew_std": 364.0235, "Agent": "SAC"}, {"env_step": 1235000, "rew": 6052.7966, "rew_std": 334.4648, "Agent": "SAC"}, {"env_step": 1240000, "rew": 6014.9591, "rew_std": 470.4387, "Agent": "SAC"}, {"env_step": 1245000, "rew": 5959.6132, "rew_std": 405.2573, "Agent": "SAC"}, {"env_step": 1250000, "rew": 6059.9079, "rew_std": 333.8162, "Agent": "SAC"}, {"env_step": 0, "rew": 990.3775, "rew_std": 4.2988, "Agent": "DDPG"}, {"env_step": 5000, "rew": -264.9603, "rew_std": 150.403, "Agent": "DDPG"}, {"env_step": 10000, "rew": -102.281, "rew_std": 91.1585, "Agent": "DDPG"}, {"env_step": 15000, "rew": -143.1314, "rew_std": 81.0776, "Agent": "DDPG"}, {"env_step": 20000, "rew": -211.1981, "rew_std": 129.6291, "Agent": "DDPG"}, {"env_step": 25000, "rew": -226.9656, "rew_std": 158.8799, "Agent": "DDPG"}, {"env_step": 30000, "rew": -224.9604, "rew_std": 104.4, "Agent": "DDPG"}, {"env_step": 35000, "rew": -372.6741, "rew_std": 211.2733, "Agent": "DDPG"}, {"env_step": 40000, "rew": -275.7362, "rew_std": 145.3991, "Agent": "DDPG"}, {"env_step": 45000, "rew": -334.0932, "rew_std": 230.6398, "Agent": "DDPG"}, {"env_step": 50000, "rew": -331.5473, "rew_std": 172.596, "Agent": "DDPG"}, {"env_step": 55000, "rew": -318.7348, "rew_std": 146.5198, "Agent": "DDPG"}, {"env_step": 60000, "rew": -439.613, "rew_std": 238.9257, "Agent": "DDPG"}, {"env_step": 65000, "rew": -285.0307, "rew_std": 192.0539, "Agent": "DDPG"}, {"env_step": 70000, "rew": -390.7666, "rew_std": 212.5538, "Agent": "DDPG"}, {"env_step": 75000, "rew": -313.1766, "rew_std": 188.3958, "Agent": "DDPG"}, {"env_step": 80000, "rew": -284.1551, "rew_std": 206.5472, "Agent": "DDPG"}, {"env_step": 85000, "rew": -199.3411, "rew_std": 71.2859, "Agent": "DDPG"}, {"env_step": 90000, "rew": -264.6298, "rew_std": 171.4911, "Agent": "DDPG"}, {"env_step": 95000, "rew": -290.3529, "rew_std": 189.5071, "Agent": "DDPG"}, {"env_step": 100000, "rew": -286.0806, "rew_std": 141.4327, "Agent": "DDPG"}, {"env_step": 105000, "rew": -262.0053, "rew_std": 160.5764, "Agent": "DDPG"}, {"env_step": 110000, "rew": -246.8272, "rew_std": 157.0853, "Agent": "DDPG"}, {"env_step": 115000, "rew": -383.9334, "rew_std": 148.8186, "Agent": "DDPG"}, {"env_step": 120000, "rew": -304.7294, "rew_std": 147.2911, "Agent": "DDPG"}, {"env_step": 125000, "rew": -351.9924, "rew_std": 249.8939, "Agent": "DDPG"}, {"env_step": 130000, "rew": -334.1367, "rew_std": 192.0608, "Agent": "DDPG"}, {"env_step": 135000, "rew": -350.9877, "rew_std": 207.3599, "Agent": "DDPG"}, {"env_step": 140000, "rew": -274.5018, "rew_std": 158.3473, "Agent": "DDPG"}, {"env_step": 145000, "rew": -323.9643, "rew_std": 171.0525, "Agent": "DDPG"}, {"env_step": 150000, "rew": -268.7093, "rew_std": 162.6873, "Agent": "DDPG"}, {"env_step": 155000, "rew": -267.165, "rew_std": 141.0928, "Agent": "DDPG"}, {"env_step": 160000, "rew": -292.6231, "rew_std": 129.7231, "Agent": "DDPG"}, {"env_step": 165000, "rew": -273.1202, "rew_std": 160.0995, "Agent": "DDPG"}, {"env_step": 170000, "rew": -229.7015, "rew_std": 94.2036, "Agent": "DDPG"}, {"env_step": 175000, "rew": -228.0526, "rew_std": 151.2225, "Agent": "DDPG"}, {"env_step": 180000, "rew": -261.7921, "rew_std": 112.1936, "Agent": "DDPG"}, {"env_step": 185000, "rew": -350.9193, "rew_std": 229.0705, "Agent": "DDPG"}, {"env_step": 190000, "rew": -344.8744, "rew_std": 189.5019, "Agent": "DDPG"}, {"env_step": 195000, "rew": -352.284, "rew_std": 245.8252, "Agent": "DDPG"}, {"env_step": 200000, "rew": -184.4852, "rew_std": 96.5308, "Agent": "DDPG"}, {"env_step": 205000, "rew": -272.4764, "rew_std": 120.4672, "Agent": "DDPG"}, {"env_step": 210000, "rew": -287.1041, "rew_std": 137.247, "Agent": "DDPG"}, {"env_step": 215000, "rew": -271.4143, "rew_std": 110.8385, "Agent": "DDPG"}, {"env_step": 220000, "rew": -272.5712, "rew_std": 119.3937, "Agent": "DDPG"}, {"env_step": 225000, "rew": -275.9917, "rew_std": 142.3404, "Agent": "DDPG"}, {"env_step": 230000, "rew": -308.7602, "rew_std": 171.4882, "Agent": "DDPG"}, {"env_step": 235000, "rew": -284.4355, "rew_std": 93.677, "Agent": "DDPG"}, {"env_step": 240000, "rew": -298.3893, "rew_std": 172.9615, "Agent": "DDPG"}, {"env_step": 245000, "rew": -387.3001, "rew_std": 216.0408, "Agent": "DDPG"}, {"env_step": 250000, "rew": -298.1023, "rew_std": 146.4381, "Agent": "DDPG"}, {"env_step": 255000, "rew": -258.6759, "rew_std": 121.7973, "Agent": "DDPG"}, {"env_step": 260000, "rew": -324.8005, "rew_std": 166.7637, "Agent": "DDPG"}, {"env_step": 265000, "rew": -255.6078, "rew_std": 156.5995, "Agent": "DDPG"}, {"env_step": 270000, "rew": -316.6506, "rew_std": 147.5006, "Agent": "DDPG"}, {"env_step": 275000, "rew": -258.3453, "rew_std": 137.476, "Agent": "DDPG"}, {"env_step": 280000, "rew": -390.4388, "rew_std": 235.748, "Agent": "DDPG"}, {"env_step": 285000, "rew": -225.9875, "rew_std": 77.8894, "Agent": "DDPG"}, {"env_step": 290000, "rew": -290.4895, "rew_std": 185.9677, "Agent": "DDPG"}, {"env_step": 295000, "rew": -300.5484, "rew_std": 123.4829, "Agent": "DDPG"}, {"env_step": 300000, "rew": -362.7267, "rew_std": 156.1688, "Agent": "DDPG"}, {"env_step": 305000, "rew": -213.9227, "rew_std": 93.6869, "Agent": "DDPG"}, {"env_step": 310000, "rew": -327.0641, "rew_std": 86.7959, "Agent": "DDPG"}, {"env_step": 315000, "rew": -234.5996, "rew_std": 132.1258, "Agent": "DDPG"}, {"env_step": 320000, "rew": -294.0503, "rew_std": 185.8398, "Agent": "DDPG"}, {"env_step": 325000, "rew": -305.9732, "rew_std": 155.4417, "Agent": "DDPG"}, {"env_step": 330000, "rew": -365.8385, "rew_std": 181.6744, "Agent": "DDPG"}, {"env_step": 335000, "rew": -368.2359, "rew_std": 182.6425, "Agent": "DDPG"}, {"env_step": 340000, "rew": -348.5352, "rew_std": 133.1437, "Agent": "DDPG"}, {"env_step": 345000, "rew": -384.662, "rew_std": 145.002, "Agent": "DDPG"}, {"env_step": 350000, "rew": -331.6225, "rew_std": 131.0167, "Agent": "DDPG"}, {"env_step": 355000, "rew": -259.168, "rew_std": 76.1548, "Agent": "DDPG"}, {"env_step": 360000, "rew": -255.8124, "rew_std": 130.9712, "Agent": "DDPG"}, {"env_step": 365000, "rew": -456.0603, "rew_std": 194.8, "Agent": "DDPG"}, {"env_step": 370000, "rew": -398.9166, "rew_std": 150.5447, "Agent": "DDPG"}, {"env_step": 375000, "rew": -405.75, "rew_std": 363.3503, "Agent": "DDPG"}, {"env_step": 380000, "rew": -662.0463, "rew_std": 177.3496, "Agent": "DDPG"}, {"env_step": 385000, "rew": -538.3942, "rew_std": 419.9355, "Agent": "DDPG"}, {"env_step": 390000, "rew": -645.4876, "rew_std": 362.0333, "Agent": "DDPG"}, {"env_step": 395000, "rew": -722.5296, "rew_std": 343.0093, "Agent": "DDPG"}, {"env_step": 400000, "rew": -785.547, "rew_std": 550.5672, "Agent": "DDPG"}, {"env_step": 405000, "rew": -651.5462, "rew_std": 359.0273, "Agent": "DDPG"}, {"env_step": 410000, "rew": -590.5508, "rew_std": 393.7796, "Agent": "DDPG"}, {"env_step": 415000, "rew": -680.8052, "rew_std": 484.7692, "Agent": "DDPG"}, {"env_step": 420000, "rew": -700.6755, "rew_std": 544.5943, "Agent": "DDPG"}, {"env_step": 425000, "rew": -570.4075, "rew_std": 538.7718, "Agent": "DDPG"}, {"env_step": 430000, "rew": -703.2318, "rew_std": 485.7277, "Agent": "DDPG"}, {"env_step": 435000, "rew": -683.4084, "rew_std": 577.0482, "Agent": "DDPG"}, {"env_step": 440000, "rew": -640.5323, "rew_std": 457.5518, "Agent": "DDPG"}, {"env_step": 445000, "rew": -574.8007, "rew_std": 472.5012, "Agent": "DDPG"}, {"env_step": 450000, "rew": -693.8662, "rew_std": 331.3535, "Agent": "DDPG"}, {"env_step": 455000, "rew": -761.8418, "rew_std": 438.3566, "Agent": "DDPG"}, {"env_step": 460000, "rew": -790.4578, "rew_std": 510.3858, "Agent": "DDPG"}, {"env_step": 465000, "rew": -734.0699, "rew_std": 502.4104, "Agent": "DDPG"}, {"env_step": 470000, "rew": -580.8933, "rew_std": 496.5509, "Agent": "DDPG"}, {"env_step": 475000, "rew": -622.7698, "rew_std": 306.7372, "Agent": "DDPG"}, {"env_step": 480000, "rew": -597.5903, "rew_std": 479.7028, "Agent": "DDPG"}, {"env_step": 485000, "rew": -496.5723, "rew_std": 409.4345, "Agent": "DDPG"}, {"env_step": 490000, "rew": -474.7281, "rew_std": 553.0422, "Agent": "DDPG"}, {"env_step": 495000, "rew": -718.454, "rew_std": 508.5459, "Agent": "DDPG"}, {"env_step": 500000, "rew": -519.0221, "rew_std": 485.7399, "Agent": "DDPG"}, {"env_step": 505000, "rew": -345.1241, "rew_std": 409.183, "Agent": "DDPG"}, {"env_step": 510000, "rew": -517.5172, "rew_std": 304.7734, "Agent": "DDPG"}, {"env_step": 515000, "rew": -387.9395, "rew_std": 315.7722, "Agent": "DDPG"}, {"env_step": 520000, "rew": -457.5481, "rew_std": 390.5131, "Agent": "DDPG"}, {"env_step": 525000, "rew": -391.5585, "rew_std": 539.0123, "Agent": "DDPG"}, {"env_step": 530000, "rew": -360.2725, "rew_std": 414.1826, "Agent": "DDPG"}, {"env_step": 535000, "rew": -320.7459, "rew_std": 465.0856, "Agent": "DDPG"}, {"env_step": 540000, "rew": -368.0916, "rew_std": 350.6185, "Agent": "DDPG"}, {"env_step": 545000, "rew": -520.4293, "rew_std": 342.2237, "Agent": "DDPG"}, {"env_step": 550000, "rew": -468.8602, "rew_std": 440.5926, "Agent": "DDPG"}, {"env_step": 555000, "rew": -378.1778, "rew_std": 502.1599, "Agent": "DDPG"}, {"env_step": 560000, "rew": -408.0724, "rew_std": 358.9013, "Agent": "DDPG"}, {"env_step": 565000, "rew": -473.7454, "rew_std": 498.5437, "Agent": "DDPG"}, {"env_step": 570000, "rew": -337.2682, "rew_std": 552.3182, "Agent": "DDPG"}, {"env_step": 575000, "rew": -318.0568, "rew_std": 663.5847, "Agent": "DDPG"}, {"env_step": 580000, "rew": -467.2081, "rew_std": 686.31, "Agent": "DDPG"}, {"env_step": 585000, "rew": -283.433, "rew_std": 499.1016, "Agent": "DDPG"}, {"env_step": 590000, "rew": -576.0506, "rew_std": 743.3304, "Agent": "DDPG"}, {"env_step": 595000, "rew": -523.4312, "rew_std": 543.8762, "Agent": "DDPG"}, {"env_step": 600000, "rew": -290.2641, "rew_std": 504.4617, "Agent": "DDPG"}, {"env_step": 605000, "rew": -287.9134, "rew_std": 468.2899, "Agent": "DDPG"}, {"env_step": 610000, "rew": -279.245, "rew_std": 510.2628, "Agent": "DDPG"}, {"env_step": 615000, "rew": -293.0744, "rew_std": 521.4399, "Agent": "DDPG"}, {"env_step": 620000, "rew": -262.9651, "rew_std": 532.2983, "Agent": "DDPG"}, {"env_step": 625000, "rew": -114.4813, "rew_std": 423.1847, "Agent": "DDPG"}, {"env_step": 630000, "rew": -252.8271, "rew_std": 655.7101, "Agent": "DDPG"}, {"env_step": 635000, "rew": -231.8681, "rew_std": 583.7409, "Agent": "DDPG"}, {"env_step": 640000, "rew": -225.1097, "rew_std": 576.478, "Agent": "DDPG"}, {"env_step": 645000, "rew": -216.8845, "rew_std": 577.7498, "Agent": "DDPG"}, {"env_step": 650000, "rew": -69.707, "rew_std": 457.7335, "Agent": "DDPG"}, {"env_step": 655000, "rew": 38.7784, "rew_std": 478.9931, "Agent": "DDPG"}, {"env_step": 660000, "rew": -48.134, "rew_std": 403.0322, "Agent": "DDPG"}, {"env_step": 665000, "rew": 74.6522, "rew_std": 390.9822, "Agent": "DDPG"}, {"env_step": 670000, "rew": 20.0033, "rew_std": 536.4488, "Agent": "DDPG"}, {"env_step": 675000, "rew": 153.8525, "rew_std": 426.2199, "Agent": "DDPG"}, {"env_step": 680000, "rew": 150.8311, "rew_std": 489.5175, "Agent": "DDPG"}, {"env_step": 685000, "rew": 37.3832, "rew_std": 762.0161, "Agent": "DDPG"}, {"env_step": 690000, "rew": 226.8652, "rew_std": 427.3436, "Agent": "DDPG"}, {"env_step": 695000, "rew": 244.6373, "rew_std": 437.9303, "Agent": "DDPG"}, {"env_step": 700000, "rew": 91.1048, "rew_std": 759.342, "Agent": "DDPG"}, {"env_step": 705000, "rew": 178.5016, "rew_std": 626.6264, "Agent": "DDPG"}, {"env_step": 710000, "rew": 108.6379, "rew_std": 898.4505, "Agent": "DDPG"}, {"env_step": 715000, "rew": 94.0608, "rew_std": 967.2715, "Agent": "DDPG"}, {"env_step": 720000, "rew": 162.4966, "rew_std": 747.3213, "Agent": "DDPG"}, {"env_step": 725000, "rew": 39.8885, "rew_std": 568.4475, "Agent": "DDPG"}, {"env_step": 730000, "rew": 52.2043, "rew_std": 812.1541, "Agent": "DDPG"}, {"env_step": 735000, "rew": 111.9638, "rew_std": 643.1707, "Agent": "DDPG"}, {"env_step": 740000, "rew": 34.238, "rew_std": 905.4223, "Agent": "DDPG"}, {"env_step": 745000, "rew": 35.7718, "rew_std": 826.4736, "Agent": "DDPG"}, {"env_step": 750000, "rew": 136.6419, "rew_std": 934.359, "Agent": "DDPG"}, {"env_step": 755000, "rew": 52.2149, "rew_std": 838.1785, "Agent": "DDPG"}, {"env_step": 760000, "rew": 104.1105, "rew_std": 826.4385, "Agent": "DDPG"}, {"env_step": 765000, "rew": 89.9899, "rew_std": 751.6642, "Agent": "DDPG"}, {"env_step": 770000, "rew": 182.7996, "rew_std": 809.7852, "Agent": "DDPG"}, {"env_step": 775000, "rew": 137.0194, "rew_std": 735.5012, "Agent": "DDPG"}, {"env_step": 780000, "rew": 183.6331, "rew_std": 557.8404, "Agent": "DDPG"}, {"env_step": 785000, "rew": 120.9358, "rew_std": 731.4159, "Agent": "DDPG"}, {"env_step": 790000, "rew": 149.1463, "rew_std": 582.7867, "Agent": "DDPG"}, {"env_step": 795000, "rew": 153.3002, "rew_std": 767.275, "Agent": "DDPG"}, {"env_step": 800000, "rew": 212.4554, "rew_std": 572.2873, "Agent": "DDPG"}, {"env_step": 805000, "rew": 186.4028, "rew_std": 478.7203, "Agent": "DDPG"}, {"env_step": 810000, "rew": 124.0634, "rew_std": 711.6635, "Agent": "DDPG"}, {"env_step": 815000, "rew": 121.4412, "rew_std": 562.7584, "Agent": "DDPG"}, {"env_step": 820000, "rew": 106.4889, "rew_std": 510.6887, "Agent": "DDPG"}, {"env_step": 825000, "rew": 201.6965, "rew_std": 494.3505, "Agent": "DDPG"}, {"env_step": 830000, "rew": 199.0214, "rew_std": 519.6556, "Agent": "DDPG"}, {"env_step": 835000, "rew": 306.1893, "rew_std": 582.493, "Agent": "DDPG"}, {"env_step": 840000, "rew": 304.76, "rew_std": 371.3446, "Agent": "DDPG"}, {"env_step": 845000, "rew": 194.2056, "rew_std": 570.5582, "Agent": "DDPG"}, {"env_step": 850000, "rew": 317.9959, "rew_std": 457.7196, "Agent": "DDPG"}, {"env_step": 855000, "rew": 313.0282, "rew_std": 375.4861, "Agent": "DDPG"}, {"env_step": 860000, "rew": 306.9646, "rew_std": 413.8683, "Agent": "DDPG"}, {"env_step": 865000, "rew": 383.4445, "rew_std": 301.2858, "Agent": "DDPG"}, {"env_step": 870000, "rew": 279.7083, "rew_std": 284.7498, "Agent": "DDPG"}, {"env_step": 875000, "rew": 358.8884, "rew_std": 412.0223, "Agent": "DDPG"}, {"env_step": 880000, "rew": 535.1728, "rew_std": 383.7316, "Agent": "DDPG"}, {"env_step": 885000, "rew": 346.6428, "rew_std": 225.1636, "Agent": "DDPG"}, {"env_step": 890000, "rew": 339.5073, "rew_std": 350.2322, "Agent": "DDPG"}, {"env_step": 895000, "rew": 331.3404, "rew_std": 685.289, "Agent": "DDPG"}, {"env_step": 900000, "rew": 346.2813, "rew_std": 604.3274, "Agent": "DDPG"}, {"env_step": 905000, "rew": 339.2269, "rew_std": 379.4845, "Agent": "DDPG"}, {"env_step": 910000, "rew": 324.2864, "rew_std": 606.8819, "Agent": "DDPG"}, {"env_step": 915000, "rew": 174.4781, "rew_std": 733.2932, "Agent": "DDPG"}, {"env_step": 920000, "rew": 255.1748, "rew_std": 575.7423, "Agent": "DDPG"}, {"env_step": 925000, "rew": 238.2941, "rew_std": 818.7228, "Agent": "DDPG"}, {"env_step": 930000, "rew": 260.7616, "rew_std": 750.9644, "Agent": "DDPG"}, {"env_step": 935000, "rew": 261.9765, "rew_std": 574.0636, "Agent": "DDPG"}, {"env_step": 940000, "rew": 159.742, "rew_std": 612.7265, "Agent": "DDPG"}, {"env_step": 945000, "rew": 470.1876, "rew_std": 621.5751, "Agent": "DDPG"}, {"env_step": 950000, "rew": 366.5236, "rew_std": 536.5978, "Agent": "DDPG"}, {"env_step": 955000, "rew": 339.7856, "rew_std": 443.0108, "Agent": "DDPG"}, {"env_step": 960000, "rew": 235.8162, "rew_std": 453.9982, "Agent": "DDPG"}, {"env_step": 965000, "rew": 229.7699, "rew_std": 661.4345, "Agent": "DDPG"}, {"env_step": 970000, "rew": 222.74, "rew_std": 424.5786, "Agent": "DDPG"}, {"env_step": 975000, "rew": 439.9229, "rew_std": 459.5935, "Agent": "DDPG"}, {"env_step": 980000, "rew": 437.2693, "rew_std": 377.1376, "Agent": "DDPG"}, {"env_step": 985000, "rew": 291.6736, "rew_std": 313.2381, "Agent": "DDPG"}, {"env_step": 990000, "rew": 469.9611, "rew_std": 448.3772, "Agent": "DDPG"}, {"env_step": 995000, "rew": 456.8604, "rew_std": 588.505, "Agent": "DDPG"}, {"env_step": 1000000, "rew": 385.1584, "rew_std": 334.4049, "Agent": "DDPG"}, {"env_step": 1005000, "rew": 551.8676, "rew_std": 488.84, "Agent": "DDPG"}, {"env_step": 1010000, "rew": 544.2, "rew_std": 328.5778, "Agent": "DDPG"}, {"env_step": 1015000, "rew": 582.8491, "rew_std": 286.5209, "Agent": "DDPG"}, {"env_step": 1020000, "rew": 484.5955, "rew_std": 273.9864, "Agent": "DDPG"}, {"env_step": 1025000, "rew": 563.1035, "rew_std": 287.232, "Agent": "DDPG"}, {"env_step": 1030000, "rew": 612.7206, "rew_std": 125.2816, "Agent": "DDPG"}, {"env_step": 1035000, "rew": 600.1382, "rew_std": 392.7771, "Agent": "DDPG"}, {"env_step": 1040000, "rew": 565.576, "rew_std": 396.5358, "Agent": "DDPG"}, {"env_step": 1045000, "rew": 487.78, "rew_std": 271.7867, "Agent": "DDPG"}, {"env_step": 1050000, "rew": 637.2793, "rew_std": 260.2136, "Agent": "DDPG"}, {"env_step": 1055000, "rew": 509.3906, "rew_std": 297.949, "Agent": "DDPG"}, {"env_step": 1060000, "rew": 583.282, "rew_std": 332.434, "Agent": "DDPG"}, {"env_step": 1065000, "rew": 601.1719, "rew_std": 217.739, "Agent": "DDPG"}, {"env_step": 1070000, "rew": 767.3685, "rew_std": 304.8501, "Agent": "DDPG"}, {"env_step": 1075000, "rew": 597.0315, "rew_std": 194.0678, "Agent": "DDPG"}, {"env_step": 1080000, "rew": 801.3029, "rew_std": 266.5503, "Agent": "DDPG"}, {"env_step": 1085000, "rew": 685.5964, "rew_std": 236.5172, "Agent": "DDPG"}, {"env_step": 1090000, "rew": 809.0626, "rew_std": 234.3018, "Agent": "DDPG"}, {"env_step": 1095000, "rew": 477.0418, "rew_std": 582.6256, "Agent": "DDPG"}, {"env_step": 1100000, "rew": 584.4327, "rew_std": 370.4071, "Agent": "DDPG"}, {"env_step": 1105000, "rew": 640.5093, "rew_std": 312.2578, "Agent": "DDPG"}, {"env_step": 1110000, "rew": 614.8422, "rew_std": 327.2431, "Agent": "DDPG"}, {"env_step": 1115000, "rew": 491.5984, "rew_std": 318.016, "Agent": "DDPG"}, {"env_step": 1120000, "rew": 538.005, "rew_std": 470.0626, "Agent": "DDPG"}, {"env_step": 1125000, "rew": 591.9692, "rew_std": 376.8948, "Agent": "DDPG"}, {"env_step": 1130000, "rew": 583.9439, "rew_std": 401.7359, "Agent": "DDPG"}, {"env_step": 1135000, "rew": 639.0098, "rew_std": 442.6075, "Agent": "DDPG"}, {"env_step": 1140000, "rew": 661.2861, "rew_std": 382.1447, "Agent": "DDPG"}, {"env_step": 1145000, "rew": 704.7731, "rew_std": 475.0417, "Agent": "DDPG"}, {"env_step": 1150000, "rew": 437.8826, "rew_std": 458.8954, "Agent": "DDPG"}, {"env_step": 1155000, "rew": 646.721, "rew_std": 467.7669, "Agent": "DDPG"}, {"env_step": 1160000, "rew": 724.8509, "rew_std": 380.5183, "Agent": "DDPG"}, {"env_step": 1165000, "rew": 694.4904, "rew_std": 368.5073, "Agent": "DDPG"}, {"env_step": 1170000, "rew": 703.0135, "rew_std": 374.4319, "Agent": "DDPG"}, {"env_step": 1175000, "rew": 726.9614, "rew_std": 465.1964, "Agent": "DDPG"}, {"env_step": 1180000, "rew": 730.3367, "rew_std": 284.869, "Agent": "DDPG"}, {"env_step": 1185000, "rew": 588.1819, "rew_std": 720.0913, "Agent": "DDPG"}, {"env_step": 1190000, "rew": 440.9309, "rew_std": 521.7464, "Agent": "DDPG"}, {"env_step": 1195000, "rew": 740.2113, "rew_std": 274.7408, "Agent": "DDPG"}, {"env_step": 1200000, "rew": 763.1286, "rew_std": 323.932, "Agent": "DDPG"}, {"env_step": 1205000, "rew": 632.2507, "rew_std": 406.6145, "Agent": "DDPG"}, {"env_step": 1210000, "rew": 751.4243, "rew_std": 444.0591, "Agent": "DDPG"}, {"env_step": 1215000, "rew": 671.4103, "rew_std": 475.4886, "Agent": "DDPG"}, {"env_step": 1220000, "rew": 818.7525, "rew_std": 396.8029, "Agent": "DDPG"}, {"env_step": 1225000, "rew": 717.9299, "rew_std": 538.4126, "Agent": "DDPG"}, {"env_step": 1230000, "rew": 658.2258, "rew_std": 608.8801, "Agent": "DDPG"}, {"env_step": 1235000, "rew": 585.0669, "rew_std": 530.2994, "Agent": "DDPG"}, {"env_step": 1240000, "rew": 772.0731, "rew_std": 481.3012, "Agent": "DDPG"}, {"env_step": 1245000, "rew": 801.4048, "rew_std": 588.8195, "Agent": "DDPG"}, {"env_step": 1250000, "rew": 615.916, "rew_std": 440.107, "Agent": "DDPG"}, {"env_step": 5000, "rew": 873.2881, "rew_std": 38.9518, "Agent": "TD3"}, {"env_step": 10000, "rew": 692.9757, "rew_std": 120.2028, "Agent": "TD3"}, {"env_step": 15000, "rew": 397.0552, "rew_std": 128.6915, "Agent": "TD3"}, {"env_step": 20000, "rew": 480.1642, "rew_std": 153.2019, "Agent": "TD3"}, {"env_step": 25000, "rew": 400.1059, "rew_std": 117.3685, "Agent": "TD3"}, {"env_step": 30000, "rew": 327.8603, "rew_std": 112.1915, "Agent": "TD3"}, {"env_step": 35000, "rew": 423.3604, "rew_std": 110.1699, "Agent": "TD3"}, {"env_step": 40000, "rew": 550.256, "rew_std": 120.7782, "Agent": "TD3"}, {"env_step": 45000, "rew": 622.4788, "rew_std": 118.6195, "Agent": "TD3"}, {"env_step": 50000, "rew": 597.6534, "rew_std": 116.1693, "Agent": "TD3"}, {"env_step": 55000, "rew": 655.5329, "rew_std": 52.1289, "Agent": "TD3"}, {"env_step": 60000, "rew": 605.0446, "rew_std": 79.2021, "Agent": "TD3"}, {"env_step": 65000, "rew": 711.6328, "rew_std": 154.2924, "Agent": "TD3"}, {"env_step": 70000, "rew": 815.7518, "rew_std": 183.6716, "Agent": "TD3"}, {"env_step": 75000, "rew": 787.1799, "rew_std": 162.4248, "Agent": "TD3"}, {"env_step": 80000, "rew": 824.0866, "rew_std": 188.9611, "Agent": "TD3"}, {"env_step": 85000, "rew": 790.719, "rew_std": 246.543, "Agent": "TD3"}, {"env_step": 90000, "rew": 919.297, "rew_std": 227.2307, "Agent": "TD3"}, {"env_step": 95000, "rew": 904.733, "rew_std": 246.6162, "Agent": "TD3"}, {"env_step": 100000, "rew": 827.7371, "rew_std": 224.2807, "Agent": "TD3"}, {"env_step": 105000, "rew": 877.6288, "rew_std": 302.8262, "Agent": "TD3"}, {"env_step": 110000, "rew": 909.9317, "rew_std": 494.907, "Agent": "TD3"}, {"env_step": 115000, "rew": 1157.1115, "rew_std": 416.5022, "Agent": "TD3"}, {"env_step": 120000, "rew": 1124.3433, "rew_std": 486.0497, "Agent": "TD3"}, {"env_step": 125000, "rew": 1267.6399, "rew_std": 570.4972, "Agent": "TD3"}, {"env_step": 130000, "rew": 1185.5972, "rew_std": 499.663, "Agent": "TD3"}, {"env_step": 135000, "rew": 1192.8429, "rew_std": 534.078, "Agent": "TD3"}, {"env_step": 140000, "rew": 1378.7758, "rew_std": 648.9205, "Agent": "TD3"}, {"env_step": 145000, "rew": 1346.322, "rew_std": 606.1947, "Agent": "TD3"}, {"env_step": 150000, "rew": 1505.2054, "rew_std": 677.1879, "Agent": "TD3"}, {"env_step": 155000, "rew": 1381.1472, "rew_std": 674.4805, "Agent": "TD3"}, {"env_step": 160000, "rew": 1399.9608, "rew_std": 402.3369, "Agent": "TD3"}, {"env_step": 165000, "rew": 1796.5644, "rew_std": 536.4386, "Agent": "TD3"}, {"env_step": 170000, "rew": 1493.7889, "rew_std": 721.5545, "Agent": "TD3"}, {"env_step": 175000, "rew": 1725.0112, "rew_std": 623.0873, "Agent": "TD3"}, {"env_step": 180000, "rew": 1938.587, "rew_std": 717.4994, "Agent": "TD3"}, {"env_step": 185000, "rew": 1783.4559, "rew_std": 740.0566, "Agent": "TD3"}, {"env_step": 190000, "rew": 1865.5029, "rew_std": 575.1663, "Agent": "TD3"}, {"env_step": 195000, "rew": 1894.567, "rew_std": 588.6718, "Agent": "TD3"}, {"env_step": 200000, "rew": 1987.107, "rew_std": 656.7583, "Agent": "TD3"}, {"env_step": 205000, "rew": 1794.3525, "rew_std": 506.857, "Agent": "TD3"}, {"env_step": 210000, "rew": 2027.3582, "rew_std": 788.0785, "Agent": "TD3"}, {"env_step": 215000, "rew": 2086.72, "rew_std": 493.8283, "Agent": "TD3"}, {"env_step": 220000, "rew": 2199.8547, "rew_std": 800.2519, "Agent": "TD3"}, {"env_step": 225000, "rew": 2274.8124, "rew_std": 698.8824, "Agent": "TD3"}, {"env_step": 230000, "rew": 2389.5561, "rew_std": 639.7828, "Agent": "TD3"}, {"env_step": 235000, "rew": 2579.1222, "rew_std": 680.182, "Agent": "TD3"}, {"env_step": 240000, "rew": 2322.6897, "rew_std": 490.1931, "Agent": "TD3"}, {"env_step": 245000, "rew": 2498.8825, "rew_std": 726.4331, "Agent": "TD3"}, {"env_step": 250000, "rew": 2523.3431, "rew_std": 643.0873, "Agent": "TD3"}, {"env_step": 255000, "rew": 2525.9347, "rew_std": 572.6054, "Agent": "TD3"}, {"env_step": 260000, "rew": 2875.4046, "rew_std": 707.0678, "Agent": "TD3"}, {"env_step": 265000, "rew": 2664.8142, "rew_std": 672.9307, "Agent": "TD3"}, {"env_step": 270000, "rew": 2917.7714, "rew_std": 714.7034, "Agent": "TD3"}, {"env_step": 275000, "rew": 2790.7163, "rew_std": 934.7828, "Agent": "TD3"}, {"env_step": 280000, "rew": 2977.8037, "rew_std": 826.6263, "Agent": "TD3"}, {"env_step": 285000, "rew": 2697.3986, "rew_std": 734.8315, "Agent": "TD3"}, {"env_step": 290000, "rew": 2907.1497, "rew_std": 863.047, "Agent": "TD3"}, {"env_step": 295000, "rew": 2967.0549, "rew_std": 708.2859, "Agent": "TD3"}, {"env_step": 300000, "rew": 2836.3506, "rew_std": 522.9356, "Agent": "TD3"}, {"env_step": 305000, "rew": 3079.2142, "rew_std": 781.9768, "Agent": "TD3"}, {"env_step": 310000, "rew": 2442.7108, "rew_std": 1035.4865, "Agent": "TD3"}, {"env_step": 315000, "rew": 2990.2987, "rew_std": 722.2599, "Agent": "TD3"}, {"env_step": 320000, "rew": 3125.9481, "rew_std": 729.2069, "Agent": "TD3"}, {"env_step": 325000, "rew": 3259.542, "rew_std": 711.0829, "Agent": "TD3"}, {"env_step": 330000, "rew": 3276.4432, "rew_std": 859.4927, "Agent": "TD3"}, {"env_step": 335000, "rew": 3186.125, "rew_std": 830.8368, "Agent": "TD3"}, {"env_step": 340000, "rew": 3258.4379, "rew_std": 710.1632, "Agent": "TD3"}, {"env_step": 345000, "rew": 3429.9357, "rew_std": 816.5659, "Agent": "TD3"}, {"env_step": 350000, "rew": 3142.0116, "rew_std": 924.3029, "Agent": "TD3"}, {"env_step": 355000, "rew": 3418.0257, "rew_std": 965.9174, "Agent": "TD3"}, {"env_step": 360000, "rew": 3423.1524, "rew_std": 903.6584, "Agent": "TD3"}, {"env_step": 365000, "rew": 3349.9063, "rew_std": 924.1522, "Agent": "TD3"}, {"env_step": 370000, "rew": 3388.3679, "rew_std": 670.0697, "Agent": "TD3"}, {"env_step": 375000, "rew": 3495.4122, "rew_std": 881.0026, "Agent": "TD3"}, {"env_step": 380000, "rew": 3522.7816, "rew_std": 834.9933, "Agent": "TD3"}, {"env_step": 385000, "rew": 3659.0542, "rew_std": 969.7367, "Agent": "TD3"}, {"env_step": 390000, "rew": 3412.5765, "rew_std": 1009.7869, "Agent": "TD3"}, {"env_step": 395000, "rew": 3441.4975, "rew_std": 828.764, "Agent": "TD3"}, {"env_step": 400000, "rew": 3507.3728, "rew_std": 1011.5694, "Agent": "TD3"}, {"env_step": 405000, "rew": 3505.6583, "rew_std": 961.0357, "Agent": "TD3"}, {"env_step": 410000, "rew": 3562.0921, "rew_std": 1000.0948, "Agent": "TD3"}, {"env_step": 415000, "rew": 3593.3924, "rew_std": 922.8568, "Agent": "TD3"}, {"env_step": 420000, "rew": 3622.2384, "rew_std": 836.6786, "Agent": "TD3"}, {"env_step": 425000, "rew": 3638.8397, "rew_std": 887.4577, "Agent": "TD3"}, {"env_step": 430000, "rew": 3628.5981, "rew_std": 902.6081, "Agent": "TD3"}, {"env_step": 435000, "rew": 3755.8288, "rew_std": 1041.5487, "Agent": "TD3"}, {"env_step": 440000, "rew": 3924.9246, "rew_std": 904.4263, "Agent": "TD3"}, {"env_step": 445000, "rew": 3848.7819, "rew_std": 863.726, "Agent": "TD3"}, {"env_step": 450000, "rew": 3786.3387, "rew_std": 881.7482, "Agent": "TD3"}, {"env_step": 455000, "rew": 3717.0026, "rew_std": 907.9821, "Agent": "TD3"}, {"env_step": 460000, "rew": 3972.8882, "rew_std": 873.4861, "Agent": "TD3"}, {"env_step": 465000, "rew": 3932.6164, "rew_std": 978.6592, "Agent": "TD3"}, {"env_step": 470000, "rew": 3856.3465, "rew_std": 963.4382, "Agent": "TD3"}, {"env_step": 475000, "rew": 3718.0162, "rew_std": 842.7566, "Agent": "TD3"}, {"env_step": 480000, "rew": 3738.2352, "rew_std": 995.4955, "Agent": "TD3"}, {"env_step": 485000, "rew": 3987.6998, "rew_std": 952.0839, "Agent": "TD3"}, {"env_step": 490000, "rew": 4052.6013, "rew_std": 783.8302, "Agent": "TD3"}, {"env_step": 495000, "rew": 3984.2972, "rew_std": 884.055, "Agent": "TD3"}, {"env_step": 500000, "rew": 3911.7144, "rew_std": 969.3004, "Agent": "TD3"}, {"env_step": 505000, "rew": 4126.2723, "rew_std": 863.1074, "Agent": "TD3"}, {"env_step": 510000, "rew": 3886.6877, "rew_std": 957.9709, "Agent": "TD3"}, {"env_step": 515000, "rew": 4062.4149, "rew_std": 1049.8531, "Agent": "TD3"}, {"env_step": 520000, "rew": 4084.531, "rew_std": 1042.5492, "Agent": "TD3"}, {"env_step": 525000, "rew": 4004.6619, "rew_std": 1103.5235, "Agent": "TD3"}, {"env_step": 530000, "rew": 3934.9948, "rew_std": 1061.6479, "Agent": "TD3"}, {"env_step": 535000, "rew": 3868.5868, "rew_std": 916.7911, "Agent": "TD3"}, {"env_step": 540000, "rew": 4259.9649, "rew_std": 779.3447, "Agent": "TD3"}, {"env_step": 545000, "rew": 4244.3096, "rew_std": 1002.53, "Agent": "TD3"}, {"env_step": 550000, "rew": 3960.2673, "rew_std": 876.5064, "Agent": "TD3"}, {"env_step": 555000, "rew": 4220.8084, "rew_std": 937.5224, "Agent": "TD3"}, {"env_step": 560000, "rew": 3938.5143, "rew_std": 820.3796, "Agent": "TD3"}, {"env_step": 565000, "rew": 4187.9774, "rew_std": 954.6565, "Agent": "TD3"}, {"env_step": 570000, "rew": 4095.1393, "rew_std": 1213.9982, "Agent": "TD3"}, {"env_step": 575000, "rew": 4315.6683, "rew_std": 871.5209, "Agent": "TD3"}, {"env_step": 580000, "rew": 4267.7739, "rew_std": 971.3827, "Agent": "TD3"}, {"env_step": 585000, "rew": 4152.9573, "rew_std": 840.2973, "Agent": "TD3"}, {"env_step": 590000, "rew": 4284.2829, "rew_std": 741.1785, "Agent": "TD3"}, {"env_step": 595000, "rew": 4348.1121, "rew_std": 908.2769, "Agent": "TD3"}, {"env_step": 600000, "rew": 4227.2577, "rew_std": 772.8629, "Agent": "TD3"}, {"env_step": 605000, "rew": 4395.6928, "rew_std": 953.3646, "Agent": "TD3"}, {"env_step": 610000, "rew": 4254.7732, "rew_std": 1073.3009, "Agent": "TD3"}, {"env_step": 615000, "rew": 4351.2641, "rew_std": 884.8591, "Agent": "TD3"}, {"env_step": 620000, "rew": 4336.1917, "rew_std": 1050.0142, "Agent": "TD3"}, {"env_step": 625000, "rew": 4127.4012, "rew_std": 835.0643, "Agent": "TD3"}, {"env_step": 630000, "rew": 4275.5443, "rew_std": 971.1633, "Agent": "TD3"}, {"env_step": 635000, "rew": 4465.4718, "rew_std": 823.02, "Agent": "TD3"}, {"env_step": 640000, "rew": 4350.7206, "rew_std": 953.1402, "Agent": "TD3"}, {"env_step": 645000, "rew": 4362.6552, "rew_std": 954.6996, "Agent": "TD3"}, {"env_step": 650000, "rew": 4301.645, "rew_std": 958.4777, "Agent": "TD3"}, {"env_step": 655000, "rew": 4432.7426, "rew_std": 960.7613, "Agent": "TD3"}, {"env_step": 660000, "rew": 4523.4884, "rew_std": 989.8137, "Agent": "TD3"}, {"env_step": 665000, "rew": 4546.4361, "rew_std": 885.5609, "Agent": "TD3"}, {"env_step": 670000, "rew": 4549.9462, "rew_std": 844.8013, "Agent": "TD3"}, {"env_step": 675000, "rew": 4321.5394, "rew_std": 910.4307, "Agent": "TD3"}, {"env_step": 680000, "rew": 4563.8536, "rew_std": 940.1484, "Agent": "TD3"}, {"env_step": 685000, "rew": 4490.2904, "rew_std": 785.6802, "Agent": "TD3"}, {"env_step": 690000, "rew": 4310.1555, "rew_std": 884.1603, "Agent": "TD3"}, {"env_step": 695000, "rew": 4326.8583, "rew_std": 1042.7645, "Agent": "TD3"}, {"env_step": 700000, "rew": 4583.9799, "rew_std": 930.0643, "Agent": "TD3"}, {"env_step": 705000, "rew": 4469.792, "rew_std": 888.5858, "Agent": "TD3"}, {"env_step": 710000, "rew": 4559.0077, "rew_std": 1086.2756, "Agent": "TD3"}, {"env_step": 715000, "rew": 4509.8791, "rew_std": 973.8389, "Agent": "TD3"}, {"env_step": 720000, "rew": 4678.2358, "rew_std": 960.0865, "Agent": "TD3"}, {"env_step": 725000, "rew": 4369.9315, "rew_std": 1196.3701, "Agent": "TD3"}, {"env_step": 730000, "rew": 4633.144, "rew_std": 854.0979, "Agent": "TD3"}, {"env_step": 735000, "rew": 4412.2788, "rew_std": 995.7851, "Agent": "TD3"}, {"env_step": 740000, "rew": 4541.2385, "rew_std": 1104.5576, "Agent": "TD3"}, {"env_step": 745000, "rew": 4534.3878, "rew_std": 795.6433, "Agent": "TD3"}, {"env_step": 750000, "rew": 4408.3879, "rew_std": 1044.1758, "Agent": "TD3"}, {"env_step": 755000, "rew": 4661.9304, "rew_std": 1098.4214, "Agent": "TD3"}, {"env_step": 760000, "rew": 4672.0089, "rew_std": 858.8352, "Agent": "TD3"}, {"env_step": 765000, "rew": 4641.7838, "rew_std": 741.3466, "Agent": "TD3"}, {"env_step": 770000, "rew": 4577.7005, "rew_std": 791.7753, "Agent": "TD3"}, {"env_step": 775000, "rew": 4645.9155, "rew_std": 916.5786, "Agent": "TD3"}, {"env_step": 780000, "rew": 4626.3724, "rew_std": 838.3194, "Agent": "TD3"}, {"env_step": 785000, "rew": 4590.6479, "rew_std": 826.8026, "Agent": "TD3"}, {"env_step": 790000, "rew": 4693.2594, "rew_std": 881.5832, "Agent": "TD3"}, {"env_step": 795000, "rew": 4571.8228, "rew_std": 1001.9763, "Agent": "TD3"}, {"env_step": 800000, "rew": 4749.5884, "rew_std": 1079.7026, "Agent": "TD3"}, {"env_step": 805000, "rew": 4749.8754, "rew_std": 895.1584, "Agent": "TD3"}, {"env_step": 810000, "rew": 4597.6498, "rew_std": 1036.8005, "Agent": "TD3"}, {"env_step": 815000, "rew": 4752.6457, "rew_std": 1035.2265, "Agent": "TD3"}, {"env_step": 820000, "rew": 4798.439, "rew_std": 832.0876, "Agent": "TD3"}, {"env_step": 825000, "rew": 4715.2744, "rew_std": 970.3636, "Agent": "TD3"}, {"env_step": 830000, "rew": 4682.6615, "rew_std": 972.6327, "Agent": "TD3"}, {"env_step": 835000, "rew": 4528.5159, "rew_std": 749.9223, "Agent": "TD3"}, {"env_step": 840000, "rew": 4867.6554, "rew_std": 895.6796, "Agent": "TD3"}, {"env_step": 845000, "rew": 4631.9192, "rew_std": 881.8808, "Agent": "TD3"}, {"env_step": 850000, "rew": 4780.6696, "rew_std": 913.4925, "Agent": "TD3"}, {"env_step": 855000, "rew": 4673.5937, "rew_std": 971.9718, "Agent": "TD3"}, {"env_step": 860000, "rew": 4731.0357, "rew_std": 968.3834, "Agent": "TD3"}, {"env_step": 865000, "rew": 4852.9469, "rew_std": 815.505, "Agent": "TD3"}, {"env_step": 870000, "rew": 4809.0254, "rew_std": 889.6001, "Agent": "TD3"}, {"env_step": 875000, "rew": 4737.5216, "rew_std": 828.7946, "Agent": "TD3"}, {"env_step": 880000, "rew": 4877.515, "rew_std": 808.7414, "Agent": "TD3"}, {"env_step": 885000, "rew": 4461.5787, "rew_std": 1001.0224, "Agent": "TD3"}, {"env_step": 890000, "rew": 4812.5231, "rew_std": 824.0173, "Agent": "TD3"}, {"env_step": 895000, "rew": 4809.0292, "rew_std": 822.1125, "Agent": "TD3"}, {"env_step": 900000, "rew": 4887.8963, "rew_std": 895.6354, "Agent": "TD3"}, {"env_step": 905000, "rew": 5053.9041, "rew_std": 762.3531, "Agent": "TD3"}, {"env_step": 910000, "rew": 4857.3797, "rew_std": 998.3682, "Agent": "TD3"}, {"env_step": 915000, "rew": 4907.1562, "rew_std": 826.4833, "Agent": "TD3"}, {"env_step": 920000, "rew": 4846.0661, "rew_std": 799.8304, "Agent": "TD3"}, {"env_step": 925000, "rew": 4923.9718, "rew_std": 725.9898, "Agent": "TD3"}, {"env_step": 930000, "rew": 4834.1895, "rew_std": 1053.2119, "Agent": "TD3"}, {"env_step": 935000, "rew": 5029.8406, "rew_std": 870.2379, "Agent": "TD3"}, {"env_step": 940000, "rew": 4828.7276, "rew_std": 772.8665, "Agent": "TD3"}, {"env_step": 945000, "rew": 4816.7274, "rew_std": 1026.5873, "Agent": "TD3"}, {"env_step": 950000, "rew": 4888.2284, "rew_std": 761.6572, "Agent": "TD3"}, {"env_step": 955000, "rew": 4804.2551, "rew_std": 899.2745, "Agent": "TD3"}, {"env_step": 960000, "rew": 4841.1225, "rew_std": 982.2515, "Agent": "TD3"}, {"env_step": 965000, "rew": 5001.7156, "rew_std": 796.2098, "Agent": "TD3"}, {"env_step": 970000, "rew": 4963.6273, "rew_std": 885.9466, "Agent": "TD3"}, {"env_step": 975000, "rew": 4873.6439, "rew_std": 934.3578, "Agent": "TD3"}, {"env_step": 980000, "rew": 4986.4284, "rew_std": 727.5026, "Agent": "TD3"}, {"env_step": 985000, "rew": 5039.04, "rew_std": 826.451, "Agent": "TD3"}, {"env_step": 990000, "rew": 4960.1642, "rew_std": 784.3563, "Agent": "TD3"}, {"env_step": 995000, "rew": 4999.292, "rew_std": 777.1722, "Agent": "TD3"}, {"env_step": 1000000, "rew": 4911.2309, "rew_std": 918.1377, "Agent": "TD3"}, {"env_step": 1005000, "rew": 4769.6997, "rew_std": 1054.2545, "Agent": "TD3"}, {"env_step": 1010000, "rew": 4880.854, "rew_std": 887.1015, "Agent": "TD3"}, {"env_step": 1015000, "rew": 4863.4448, "rew_std": 950.289, "Agent": "TD3"}, {"env_step": 1020000, "rew": 5054.0226, "rew_std": 812.4345, "Agent": "TD3"}, {"env_step": 1025000, "rew": 4866.6239, "rew_std": 849.4627, "Agent": "TD3"}, {"env_step": 1030000, "rew": 4781.6789, "rew_std": 846.0881, "Agent": "TD3"}, {"env_step": 1035000, "rew": 5037.8187, "rew_std": 904.3878, "Agent": "TD3"}, {"env_step": 1040000, "rew": 5010.1024, "rew_std": 883.645, "Agent": "TD3"}, {"env_step": 1045000, "rew": 4940.0185, "rew_std": 853.7942, "Agent": "TD3"}, {"env_step": 1050000, "rew": 4869.8351, "rew_std": 1065.8683, "Agent": "TD3"}, {"env_step": 1055000, "rew": 5065.608, "rew_std": 884.4816, "Agent": "TD3"}, {"env_step": 1060000, "rew": 5079.3512, "rew_std": 797.8488, "Agent": "TD3"}, {"env_step": 1065000, "rew": 5050.3343, "rew_std": 703.362, "Agent": "TD3"}, {"env_step": 1070000, "rew": 5188.6333, "rew_std": 883.3329, "Agent": "TD3"}, {"env_step": 1075000, "rew": 4966.898, "rew_std": 924.0908, "Agent": "TD3"}, {"env_step": 1080000, "rew": 5028.612, "rew_std": 910.7201, "Agent": "TD3"}, {"env_step": 1085000, "rew": 5146.3244, "rew_std": 829.8989, "Agent": "TD3"}, {"env_step": 1090000, "rew": 5095.4587, "rew_std": 881.6364, "Agent": "TD3"}, {"env_step": 1095000, "rew": 5087.68, "rew_std": 1029.9957, "Agent": "TD3"}, {"env_step": 1100000, "rew": 5165.735, "rew_std": 757.8112, "Agent": "TD3"}, {"env_step": 1105000, "rew": 5180.7916, "rew_std": 924.8973, "Agent": "TD3"}, {"env_step": 1110000, "rew": 5258.8776, "rew_std": 723.154, "Agent": "TD3"}, {"env_step": 1115000, "rew": 5154.2468, "rew_std": 946.6712, "Agent": "TD3"}, {"env_step": 1120000, "rew": 4855.1916, "rew_std": 998.1512, "Agent": "TD3"}, {"env_step": 1125000, "rew": 5109.7468, "rew_std": 866.5871, "Agent": "TD3"}, {"env_step": 1130000, "rew": 5109.2571, "rew_std": 875.319, "Agent": "TD3"}, {"env_step": 1135000, "rew": 5249.2797, "rew_std": 817.1159, "Agent": "TD3"}, {"env_step": 1140000, "rew": 5183.5445, "rew_std": 876.9945, "Agent": "TD3"}, {"env_step": 1145000, "rew": 5111.9414, "rew_std": 935.2839, "Agent": "TD3"}, {"env_step": 1150000, "rew": 5140.7273, "rew_std": 805.8536, "Agent": "TD3"}, {"env_step": 1155000, "rew": 5037.2696, "rew_std": 845.695, "Agent": "TD3"}, {"env_step": 1160000, "rew": 5008.4221, "rew_std": 952.8048, "Agent": "TD3"}, {"env_step": 1165000, "rew": 5130.9812, "rew_std": 991.83, "Agent": "TD3"}, {"env_step": 1170000, "rew": 4889.7251, "rew_std": 1224.0601, "Agent": "TD3"}, {"env_step": 1175000, "rew": 4713.3788, "rew_std": 1632.4869, "Agent": "TD3"}, {"env_step": 1180000, "rew": 4964.1914, "rew_std": 1026.0415, "Agent": "TD3"}, {"env_step": 1185000, "rew": 4920.8587, "rew_std": 1037.1662, "Agent": "TD3"}, {"env_step": 1190000, "rew": 4827.8426, "rew_std": 995.3367, "Agent": "TD3"}, {"env_step": 1195000, "rew": 5161.0952, "rew_std": 945.0446, "Agent": "TD3"}, {"env_step": 1200000, "rew": 5250.055, "rew_std": 834.4152, "Agent": "TD3"}, {"env_step": 1205000, "rew": 5257.3788, "rew_std": 779.9084, "Agent": "TD3"}, {"env_step": 1210000, "rew": 4970.1045, "rew_std": 974.5391, "Agent": "TD3"}, {"env_step": 1215000, "rew": 4639.1005, "rew_std": 1732.0484, "Agent": "TD3"}, {"env_step": 1220000, "rew": 4353.0499, "rew_std": 2244.2511, "Agent": "TD3"}, {"env_step": 1225000, "rew": 4075.511, "rew_std": 2314.6058, "Agent": "TD3"}, {"env_step": 1230000, "rew": 5068.5288, "rew_std": 991.9342, "Agent": "TD3"}, {"env_step": 1235000, "rew": 5115.0143, "rew_std": 878.5611, "Agent": "TD3"}, {"env_step": 1240000, "rew": 5238.8184, "rew_std": 1062.6486, "Agent": "TD3"}, {"env_step": 1245000, "rew": 5198.4506, "rew_std": 794.2966, "Agent": "TD3"}, {"env_step": 1250000, "rew": 5433.2663, "rew_std": 805.9253, "Agent": "TD3"}, {"env_step": 0, "rew": -46.1947, "rew_std": 25.5804, "Agent": "TRPO"}, {"env_step": 30720, "rew": -13.9012, "rew_std": 11.913, "Agent": "TRPO"}, {"env_step": 61440, "rew": 18.0615, "rew_std": 9.0166, "Agent": "TRPO"}, {"env_step": 92160, "rew": 51.9175, "rew_std": 16.4955, "Agent": "TRPO"}, {"env_step": 122880, "rew": 108.4958, "rew_std": 33.2241, "Agent": "TRPO"}, {"env_step": 153600, "rew": 183.9944, "rew_std": 35.8262, "Agent": "TRPO"}, {"env_step": 184320, "rew": 257.4861, "rew_std": 60.051, "Agent": "TRPO"}, {"env_step": 215040, "rew": 430.4068, "rew_std": 124.2541, "Agent": "TRPO"}, {"env_step": 245760, "rew": 564.2017, "rew_std": 146.0606, "Agent": "TRPO"}, {"env_step": 276480, "rew": 682.5168, "rew_std": 122.3187, "Agent": "TRPO"}, {"env_step": 307200, "rew": 832.5033, "rew_std": 159.2968, "Agent": "TRPO"}, {"env_step": 337920, "rew": 1014.1049, "rew_std": 221.8108, "Agent": "TRPO"}, {"env_step": 368640, "rew": 1128.5618, "rew_std": 167.9213, "Agent": "TRPO"}, {"env_step": 399360, "rew": 1146.4494, "rew_std": 215.1602, "Agent": "TRPO"}, {"env_step": 430080, "rew": 1233.6374, "rew_std": 286.8604, "Agent": "TRPO"}, {"env_step": 460800, "rew": 1261.4876, "rew_std": 301.0892, "Agent": "TRPO"}, {"env_step": 491520, "rew": 1472.6865, "rew_std": 215.7148, "Agent": "TRPO"}, {"env_step": 522240, "rew": 1501.5343, "rew_std": 428.1053, "Agent": "TRPO"}, {"env_step": 552960, "rew": 1631.5161, "rew_std": 558.1692, "Agent": "TRPO"}, {"env_step": 583680, "rew": 1741.6005, "rew_std": 483.7599, "Agent": "TRPO"}, {"env_step": 614400, "rew": 1809.3667, "rew_std": 444.8329, "Agent": "TRPO"}, {"env_step": 645120, "rew": 1943.2351, "rew_std": 386.0455, "Agent": "TRPO"}, {"env_step": 675840, "rew": 2119.3593, "rew_std": 463.9853, "Agent": "TRPO"}, {"env_step": 706560, "rew": 2160.9349, "rew_std": 299.8534, "Agent": "TRPO"}, {"env_step": 737280, "rew": 2205.6584, "rew_std": 495.8226, "Agent": "TRPO"}, {"env_step": 768000, "rew": 2148.4616, "rew_std": 617.4263, "Agent": "TRPO"}, {"env_step": 798720, "rew": 2294.2347, "rew_std": 619.654, "Agent": "TRPO"}, {"env_step": 829440, "rew": 2334.3877, "rew_std": 569.8366, "Agent": "TRPO"}, {"env_step": 860160, "rew": 2373.0195, "rew_std": 635.4872, "Agent": "TRPO"}, {"env_step": 890880, "rew": 2572.1874, "rew_std": 771.2781, "Agent": "TRPO"}, {"env_step": 921600, "rew": 2568.2581, "rew_std": 539.7726, "Agent": "TRPO"}, {"env_step": 952320, "rew": 2639.2867, "rew_std": 572.4301, "Agent": "TRPO"}, {"env_step": 983040, "rew": 2866.6508, "rew_std": 707.8937, "Agent": "TRPO"}, {"env_step": 1013760, "rew": 2731.0697, "rew_std": 675.644, "Agent": "TRPO"}, {"env_step": 1044480, "rew": 2702.1965, "rew_std": 742.4807, "Agent": "TRPO"}, {"env_step": 1075200, "rew": 3001.7621, "rew_std": 688.8314, "Agent": "TRPO"}, {"env_step": 1105920, "rew": 2877.8716, "rew_std": 922.2411, "Agent": "TRPO"}, {"env_step": 1136640, "rew": 3104.9072, "rew_std": 800.8227, "Agent": "TRPO"}, {"env_step": 1167360, "rew": 2976.5528, "rew_std": 661.5153, "Agent": "TRPO"}, {"env_step": 1198080, "rew": 3196.2228, "rew_std": 707.9679, "Agent": "TRPO"}, {"env_step": 1228800, "rew": 3237.2005, "rew_std": 505.6685, "Agent": "TRPO"}, {"env_step": 1259520, "rew": 3098.3083, "rew_std": 578.4346, "Agent": "TRPO"}, {"env_step": 1290240, "rew": 3356.962, "rew_std": 786.3838, "Agent": "TRPO"}, {"env_step": 1320960, "rew": 3241.2647, "rew_std": 600.3495, "Agent": "TRPO"}, {"env_step": 1351680, "rew": 3361.1722, "rew_std": 547.5151, "Agent": "TRPO"}, {"env_step": 1382400, "rew": 3495.3879, "rew_std": 662.5387, "Agent": "TRPO"}, {"env_step": 1413120, "rew": 3388.8443, "rew_std": 537.1016, "Agent": "TRPO"}, {"env_step": 1443840, "rew": 3493.6456, "rew_std": 509.2391, "Agent": "TRPO"}, {"env_step": 1474560, "rew": 3256.3625, "rew_std": 412.1903, "Agent": "TRPO"}, {"env_step": 1505280, "rew": 3695.3862, "rew_std": 644.5284, "Agent": "TRPO"}, {"env_step": 1536000, "rew": 3685.0253, "rew_std": 537.4812, "Agent": "TRPO"}, {"env_step": 1566720, "rew": 3565.3736, "rew_std": 647.2313, "Agent": "TRPO"}, {"env_step": 1597440, "rew": 3790.5153, "rew_std": 702.1943, "Agent": "TRPO"}, {"env_step": 1628160, "rew": 3598.3267, "rew_std": 626.2541, "Agent": "TRPO"}, {"env_step": 1658880, "rew": 3970.7287, "rew_std": 647.9836, "Agent": "TRPO"}, {"env_step": 1689600, "rew": 3658.6492, "rew_std": 645.0697, "Agent": "TRPO"}, {"env_step": 1720320, "rew": 3848.0345, "rew_std": 701.1273, "Agent": "TRPO"}, {"env_step": 1751040, "rew": 4061.4513, "rew_std": 690.0289, "Agent": "TRPO"}, {"env_step": 1781760, "rew": 3807.921, "rew_std": 706.5999, "Agent": "TRPO"}, {"env_step": 1812480, "rew": 3802.0342, "rew_std": 663.7139, "Agent": "TRPO"}, {"env_step": 1843200, "rew": 3873.6662, "rew_std": 566.3833, "Agent": "TRPO"}, {"env_step": 1873920, "rew": 4038.6082, "rew_std": 518.5079, "Agent": "TRPO"}, {"env_step": 1904640, "rew": 4167.3221, "rew_std": 440.9658, "Agent": "TRPO"}, {"env_step": 1935360, "rew": 3922.2855, "rew_std": 733.8238, "Agent": "TRPO"}, {"env_step": 1966080, "rew": 4174.7307, "rew_std": 495.1849, "Agent": "TRPO"}, {"env_step": 1996800, "rew": 4155.8384, "rew_std": 477.6376, "Agent": "TRPO"}, {"env_step": 2027520, "rew": 4259.3055, "rew_std": 577.3636, "Agent": "TRPO"}, {"env_step": 2058240, "rew": 4256.1933, "rew_std": 594.2826, "Agent": "TRPO"}, {"env_step": 2088960, "rew": 4355.0669, "rew_std": 605.8978, "Agent": "TRPO"}, {"env_step": 2119680, "rew": 4297.2252, "rew_std": 474.421, "Agent": "TRPO"}, {"env_step": 2150400, "rew": 4287.4698, "rew_std": 363.7332, "Agent": "TRPO"}, {"env_step": 2181120, "rew": 4398.2137, "rew_std": 354.3603, "Agent": "TRPO"}, {"env_step": 2211840, "rew": 4413.5796, "rew_std": 460.9681, "Agent": "TRPO"}, {"env_step": 2242560, "rew": 4329.001, "rew_std": 530.2755, "Agent": "TRPO"}, {"env_step": 2273280, "rew": 4252.3464, "rew_std": 597.5882, "Agent": "TRPO"}, {"env_step": 2304000, "rew": 4462.5657, "rew_std": 495.3802, "Agent": "TRPO"}, {"env_step": 2334720, "rew": 4374.2838, "rew_std": 457.0956, "Agent": "TRPO"}, {"env_step": 2365440, "rew": 4529.4549, "rew_std": 461.1733, "Agent": "TRPO"}, {"env_step": 2396160, "rew": 4291.4468, "rew_std": 411.2142, "Agent": "TRPO"}, {"env_step": 2426880, "rew": 4430.1459, "rew_std": 601.8812, "Agent": "TRPO"}, {"env_step": 2457600, "rew": 4508.078, "rew_std": 465.0507, "Agent": "TRPO"}, {"env_step": 2488320, "rew": 4151.3796, "rew_std": 414.656, "Agent": "TRPO"}, {"env_step": 2519040, "rew": 4705.7212, "rew_std": 477.1206, "Agent": "TRPO"}, {"env_step": 2549760, "rew": 4641.3111, "rew_std": 622.6369, "Agent": "TRPO"}, {"env_step": 2580480, "rew": 4484.1452, "rew_std": 442.0811, "Agent": "TRPO"}, {"env_step": 2611200, "rew": 4658.7294, "rew_std": 432.1496, "Agent": "TRPO"}, {"env_step": 2641920, "rew": 4625.8785, "rew_std": 357.4208, "Agent": "TRPO"}, {"env_step": 2672640, "rew": 4284.9794, "rew_std": 599.7262, "Agent": "TRPO"}, {"env_step": 2703360, "rew": 4419.1972, "rew_std": 447.5325, "Agent": "TRPO"}, {"env_step": 2734080, "rew": 4746.1067, "rew_std": 580.4515, "Agent": "TRPO"}, {"env_step": 2764800, "rew": 4756.6644, "rew_std": 502.4658, "Agent": "TRPO"}, {"env_step": 2795520, "rew": 4607.7287, "rew_std": 427.1984, "Agent": "TRPO"}, {"env_step": 2826240, "rew": 4404.4363, "rew_std": 555.3297, "Agent": "TRPO"}, {"env_step": 2856960, "rew": 4816.8623, "rew_std": 528.6794, "Agent": "TRPO"}, {"env_step": 2887680, "rew": 4753.3578, "rew_std": 715.7057, "Agent": "TRPO"}, {"env_step": 2918400, "rew": 4604.763, "rew_std": 447.4458, "Agent": "TRPO"}, {"env_step": 2949120, "rew": 4800.2693, "rew_std": 602.9667, "Agent": "TRPO"}, {"env_step": 2979840, "rew": 4772.9396, "rew_std": 368.1691, "Agent": "TRPO"}, {"env_step": 3010560, "rew": 4897.0257, "rew_std": 449.1329, "Agent": "TRPO"}, {"env_step": 3041280, "rew": 4841.7814, "rew_std": 534.8951, "Agent": "TRPO"}, {"env_step": 3072000, "rew": 4604.9043, "rew_std": 580.976, "Agent": "TRPO"}, {"env_step": 0, "rew": -46.1947, "rew_std": 25.5804, "Agent": "NPG"}, {"env_step": 30720, "rew": -29.0009, "rew_std": 15.8916, "Agent": "NPG"}, {"env_step": 61440, "rew": -10.4279, "rew_std": 11.0242, "Agent": "NPG"}, {"env_step": 92160, "rew": 11.1187, "rew_std": 9.0737, "Agent": "NPG"}, {"env_step": 122880, "rew": 27.5852, "rew_std": 16.4638, "Agent": "NPG"}, {"env_step": 153600, "rew": 56.7285, "rew_std": 23.8642, "Agent": "NPG"}, {"env_step": 184320, "rew": 81.2319, "rew_std": 29.1525, "Agent": "NPG"}, {"env_step": 215040, "rew": 159.4704, "rew_std": 47.3793, "Agent": "NPG"}, {"env_step": 245760, "rew": 174.9781, "rew_std": 57.693, "Agent": "NPG"}, {"env_step": 276480, "rew": 178.6358, "rew_std": 32.807, "Agent": "NPG"}, {"env_step": 307200, "rew": 218.3212, "rew_std": 41.035, "Agent": "NPG"}, {"env_step": 337920, "rew": 368.663, "rew_std": 80.0623, "Agent": "NPG"}, {"env_step": 368640, "rew": 432.2163, "rew_std": 63.1225, "Agent": "NPG"}, {"env_step": 399360, "rew": 568.0807, "rew_std": 125.517, "Agent": "NPG"}, {"env_step": 430080, "rew": 631.1259, "rew_std": 114.6371, "Agent": "NPG"}, {"env_step": 460800, "rew": 753.5929, "rew_std": 118.5185, "Agent": "NPG"}, {"env_step": 491520, "rew": 864.7371, "rew_std": 183.4071, "Agent": "NPG"}, {"env_step": 522240, "rew": 940.2978, "rew_std": 158.8018, "Agent": "NPG"}, {"env_step": 552960, "rew": 1021.7661, "rew_std": 159.2406, "Agent": "NPG"}, {"env_step": 583680, "rew": 1131.714, "rew_std": 150.4756, "Agent": "NPG"}, {"env_step": 614400, "rew": 1201.5181, "rew_std": 164.258, "Agent": "NPG"}, {"env_step": 645120, "rew": 1343.1191, "rew_std": 184.6686, "Agent": "NPG"}, {"env_step": 675840, "rew": 1318.401, "rew_std": 229.7468, "Agent": "NPG"}, {"env_step": 706560, "rew": 1459.1159, "rew_std": 222.6298, "Agent": "NPG"}, {"env_step": 737280, "rew": 1542.0295, "rew_std": 203.9479, "Agent": "NPG"}, {"env_step": 768000, "rew": 1634.9176, "rew_std": 218.2131, "Agent": "NPG"}, {"env_step": 798720, "rew": 1692.0983, "rew_std": 343.9785, "Agent": "NPG"}, {"env_step": 829440, "rew": 1906.1507, "rew_std": 457.5102, "Agent": "NPG"}, {"env_step": 860160, "rew": 1775.1748, "rew_std": 342.0144, "Agent": "NPG"}, {"env_step": 890880, "rew": 2102.0494, "rew_std": 355.8503, "Agent": "NPG"}, {"env_step": 921600, "rew": 2046.7794, "rew_std": 340.8884, "Agent": "NPG"}, {"env_step": 952320, "rew": 2164.3512, "rew_std": 405.7482, "Agent": "NPG"}, {"env_step": 983040, "rew": 2348.312, "rew_std": 509.9401, "Agent": "NPG"}, {"env_step": 1013760, "rew": 2365.8958, "rew_std": 523.6603, "Agent": "NPG"}, {"env_step": 1044480, "rew": 2324.9164, "rew_std": 528.4526, "Agent": "NPG"}, {"env_step": 1075200, "rew": 2493.722, "rew_std": 583.8895, "Agent": "NPG"}, {"env_step": 1105920, "rew": 2638.7282, "rew_std": 650.3356, "Agent": "NPG"}, {"env_step": 1136640, "rew": 2588.3598, "rew_std": 506.4239, "Agent": "NPG"}, {"env_step": 1167360, "rew": 2657.8218, "rew_std": 542.1361, "Agent": "NPG"}, {"env_step": 1198080, "rew": 2772.0887, "rew_std": 399.8722, "Agent": "NPG"}, {"env_step": 1228800, "rew": 2983.8532, "rew_std": 405.1427, "Agent": "NPG"}, {"env_step": 1259520, "rew": 2969.912, "rew_std": 543.0224, "Agent": "NPG"}, {"env_step": 1290240, "rew": 2898.5856, "rew_std": 687.1927, "Agent": "NPG"}, {"env_step": 1320960, "rew": 2979.687, "rew_std": 685.3181, "Agent": "NPG"}, {"env_step": 1351680, "rew": 2937.7865, "rew_std": 654.2811, "Agent": "NPG"}, {"env_step": 1382400, "rew": 3180.3262, "rew_std": 502.1675, "Agent": "NPG"}, {"env_step": 1413120, "rew": 3181.4746, "rew_std": 526.9918, "Agent": "NPG"}, {"env_step": 1443840, "rew": 3186.725, "rew_std": 483.0104, "Agent": "NPG"}, {"env_step": 1474560, "rew": 3347.1624, "rew_std": 474.3212, "Agent": "NPG"}, {"env_step": 1505280, "rew": 3264.1247, "rew_std": 542.1326, "Agent": "NPG"}, {"env_step": 1536000, "rew": 3267.3737, "rew_std": 536.477, "Agent": "NPG"}, {"env_step": 1566720, "rew": 3677.709, "rew_std": 486.0857, "Agent": "NPG"}, {"env_step": 1597440, "rew": 3379.1083, "rew_std": 683.1763, "Agent": "NPG"}, {"env_step": 1628160, "rew": 3533.6653, "rew_std": 437.9623, "Agent": "NPG"}, {"env_step": 1658880, "rew": 3534.0292, "rew_std": 787.6867, "Agent": "NPG"}, {"env_step": 1689600, "rew": 3399.7589, "rew_std": 685.1419, "Agent": "NPG"}, {"env_step": 1720320, "rew": 3587.7024, "rew_std": 471.134, "Agent": "NPG"}, {"env_step": 1751040, "rew": 3560.6233, "rew_std": 616.8093, "Agent": "NPG"}, {"env_step": 1781760, "rew": 3635.7303, "rew_std": 608.4666, "Agent": "NPG"}, {"env_step": 1812480, "rew": 3713.5899, "rew_std": 802.8577, "Agent": "NPG"}, {"env_step": 1843200, "rew": 3590.908, "rew_std": 557.7779, "Agent": "NPG"}, {"env_step": 1873920, "rew": 3903.7812, "rew_std": 655.7805, "Agent": "NPG"}, {"env_step": 1904640, "rew": 3770.5075, "rew_std": 645.9229, "Agent": "NPG"}, {"env_step": 1935360, "rew": 4110.3599, "rew_std": 657.3085, "Agent": "NPG"}, {"env_step": 1966080, "rew": 3851.8914, "rew_std": 745.2079, "Agent": "NPG"}, {"env_step": 1996800, "rew": 3998.0014, "rew_std": 650.7886, "Agent": "NPG"}, {"env_step": 2027520, "rew": 3739.3657, "rew_std": 715.6727, "Agent": "NPG"}, {"env_step": 2058240, "rew": 4230.5764, "rew_std": 578.5901, "Agent": "NPG"}, {"env_step": 2088960, "rew": 4140.935, "rew_std": 624.3245, "Agent": "NPG"}, {"env_step": 2119680, "rew": 4228.9808, "rew_std": 573.9637, "Agent": "NPG"}, {"env_step": 2150400, "rew": 4142.6649, "rew_std": 419.5781, "Agent": "NPG"}, {"env_step": 2181120, "rew": 4089.8147, "rew_std": 302.9776, "Agent": "NPG"}, {"env_step": 2211840, "rew": 4276.8854, "rew_std": 674.4424, "Agent": "NPG"}, {"env_step": 2242560, "rew": 4312.1654, "rew_std": 567.5481, "Agent": "NPG"}, {"env_step": 2273280, "rew": 4224.3566, "rew_std": 445.0954, "Agent": "NPG"}, {"env_step": 2304000, "rew": 3878.7632, "rew_std": 866.3551, "Agent": "NPG"}, {"env_step": 2334720, "rew": 4001.3801, "rew_std": 478.1705, "Agent": "NPG"}, {"env_step": 2365440, "rew": 4284.5594, "rew_std": 448.8121, "Agent": "NPG"}, {"env_step": 2396160, "rew": 4406.6923, "rew_std": 516.8683, "Agent": "NPG"}, {"env_step": 2426880, "rew": 4145.9032, "rew_std": 519.4268, "Agent": "NPG"}, {"env_step": 2457600, "rew": 4241.7632, "rew_std": 685.6791, "Agent": "NPG"}, {"env_step": 2488320, "rew": 4154.1219, "rew_std": 437.1567, "Agent": "NPG"}, {"env_step": 2519040, "rew": 4495.7374, "rew_std": 449.1806, "Agent": "NPG"}, {"env_step": 2549760, "rew": 4289.7487, "rew_std": 526.7084, "Agent": "NPG"}, {"env_step": 2580480, "rew": 4246.0211, "rew_std": 530.6265, "Agent": "NPG"}, {"env_step": 2611200, "rew": 4463.8985, "rew_std": 572.8797, "Agent": "NPG"}, {"env_step": 2641920, "rew": 4298.6282, "rew_std": 665.9416, "Agent": "NPG"}, {"env_step": 2672640, "rew": 4485.5394, "rew_std": 359.9158, "Agent": "NPG"}, {"env_step": 2703360, "rew": 4820.597, "rew_std": 318.5627, "Agent": "NPG"}, {"env_step": 2734080, "rew": 4540.1727, "rew_std": 412.9545, "Agent": "NPG"}, {"env_step": 2764800, "rew": 4500.1939, "rew_std": 596.1343, "Agent": "NPG"}, {"env_step": 2795520, "rew": 4664.0692, "rew_std": 426.6286, "Agent": "NPG"}, {"env_step": 2826240, "rew": 4516.467, "rew_std": 435.656, "Agent": "NPG"}, {"env_step": 2856960, "rew": 4545.139, "rew_std": 496.411, "Agent": "NPG"}, {"env_step": 2887680, "rew": 4444.0518, "rew_std": 596.6213, "Agent": "NPG"}, {"env_step": 2918400, "rew": 4658.8307, "rew_std": 652.0561, "Agent": "NPG"}, {"env_step": 2949120, "rew": 4509.4439, "rew_std": 466.3483, "Agent": "NPG"}, {"env_step": 2979840, "rew": 4589.8375, "rew_std": 414.0363, "Agent": "NPG"}, {"env_step": 3010560, "rew": 4693.0902, "rew_std": 490.4674, "Agent": "NPG"}, {"env_step": 3041280, "rew": 4678.8152, "rew_std": 313.0054, "Agent": "NPG"}, {"env_step": 3072000, "rew": 4736.6273, "rew_std": 376.7205, "Agent": "NPG"}, {"env_step": 0, "rew": -45.1638, "rew_std": 26.8049, "Agent": "PPO"}, {"env_step": 30720, "rew": 1.1798, "rew_std": 7.6052, "Agent": "PPO"}, {"env_step": 61440, "rew": 108.396, "rew_std": 24.5339, "Agent": "PPO"}, {"env_step": 92160, "rew": 257.0256, "rew_std": 60.1797, "Agent": "PPO"}, {"env_step": 122880, "rew": 409.4479, "rew_std": 107.6753, "Agent": "PPO"}, {"env_step": 153600, "rew": 576.9856, "rew_std": 104.0574, "Agent": "PPO"}, {"env_step": 184320, "rew": 680.1085, "rew_std": 154.5434, "Agent": "PPO"}, {"env_step": 215040, "rew": 734.055, "rew_std": 157.4357, "Agent": "PPO"}, {"env_step": 245760, "rew": 902.7627, "rew_std": 256.8418, "Agent": "PPO"}, {"env_step": 276480, "rew": 1002.7442, "rew_std": 263.7889, "Agent": "PPO"}, {"env_step": 307200, "rew": 933.2541, "rew_std": 233.041, "Agent": "PPO"}, {"env_step": 337920, "rew": 1381.7994, "rew_std": 171.3465, "Agent": "PPO"}, {"env_step": 368640, "rew": 1365.8795, "rew_std": 265.6691, "Agent": "PPO"}, {"env_step": 399360, "rew": 1422.151, "rew_std": 355.0902, "Agent": "PPO"}, {"env_step": 430080, "rew": 1437.5237, "rew_std": 351.6675, "Agent": "PPO"}, {"env_step": 460800, "rew": 1592.5595, "rew_std": 435.5349, "Agent": "PPO"}, {"env_step": 491520, "rew": 1830.9248, "rew_std": 327.1536, "Agent": "PPO"}, {"env_step": 522240, "rew": 1865.1769, "rew_std": 617.9661, "Agent": "PPO"}, {"env_step": 552960, "rew": 2027.5555, "rew_std": 450.231, "Agent": "PPO"}, {"env_step": 583680, "rew": 1922.7912, "rew_std": 530.9554, "Agent": "PPO"}, {"env_step": 614400, "rew": 2155.52, "rew_std": 605.2654, "Agent": "PPO"}, {"env_step": 645120, "rew": 2281.4225, "rew_std": 659.161, "Agent": "PPO"}, {"env_step": 675840, "rew": 2466.7854, "rew_std": 800.2599, "Agent": "PPO"}, {"env_step": 706560, "rew": 2778.6914, "rew_std": 818.3568, "Agent": "PPO"}, {"env_step": 737280, "rew": 2408.4636, "rew_std": 658.6381, "Agent": "PPO"}, {"env_step": 768000, "rew": 2787.5943, "rew_std": 758.873, "Agent": "PPO"}, {"env_step": 798720, "rew": 2672.0722, "rew_std": 886.4375, "Agent": "PPO"}, {"env_step": 829440, "rew": 2815.8244, "rew_std": 756.2206, "Agent": "PPO"}, {"env_step": 860160, "rew": 2881.8245, "rew_std": 932.4833, "Agent": "PPO"}, {"env_step": 890880, "rew": 3054.5044, "rew_std": 656.7209, "Agent": "PPO"}, {"env_step": 921600, "rew": 2805.7394, "rew_std": 811.9402, "Agent": "PPO"}, {"env_step": 952320, "rew": 3156.5113, "rew_std": 882.0281, "Agent": "PPO"}, {"env_step": 983040, "rew": 3258.3684, "rew_std": 1079.2574, "Agent": "PPO"}, {"env_step": 1013760, "rew": 3141.0442, "rew_std": 1009.4955, "Agent": "PPO"}, {"env_step": 1044480, "rew": 3103.7708, "rew_std": 785.5033, "Agent": "PPO"}, {"env_step": 1075200, "rew": 2894.7434, "rew_std": 844.0465, "Agent": "PPO"}, {"env_step": 1105920, "rew": 3268.2543, "rew_std": 849.4982, "Agent": "PPO"}, {"env_step": 1136640, "rew": 3032.4115, "rew_std": 887.7795, "Agent": "PPO"}, {"env_step": 1167360, "rew": 2798.2882, "rew_std": 1018.4211, "Agent": "PPO"}, {"env_step": 1198080, "rew": 2982.8664, "rew_std": 1182.9198, "Agent": "PPO"}, {"env_step": 1228800, "rew": 2853.1494, "rew_std": 856.6556, "Agent": "PPO"}, {"env_step": 1259520, "rew": 3189.8264, "rew_std": 857.3083, "Agent": "PPO"}, {"env_step": 1290240, "rew": 3148.1075, "rew_std": 1157.6515, "Agent": "PPO"}, {"env_step": 1320960, "rew": 3414.5263, "rew_std": 1133.0382, "Agent": "PPO"}, {"env_step": 1351680, "rew": 3182.4456, "rew_std": 848.7621, "Agent": "PPO"}, {"env_step": 1382400, "rew": 3163.9297, "rew_std": 898.0596, "Agent": "PPO"}, {"env_step": 1413120, "rew": 3334.0317, "rew_std": 1050.2921, "Agent": "PPO"}, {"env_step": 1443840, "rew": 3112.8048, "rew_std": 1262.7364, "Agent": "PPO"}, {"env_step": 1474560, "rew": 3406.995, "rew_std": 1065.5177, "Agent": "PPO"}, {"env_step": 1505280, "rew": 3200.8696, "rew_std": 1149.5605, "Agent": "PPO"}, {"env_step": 1536000, "rew": 3336.6813, "rew_std": 867.9635, "Agent": "PPO"}, {"env_step": 1566720, "rew": 3522.898, "rew_std": 1000.9456, "Agent": "PPO"}, {"env_step": 1597440, "rew": 3435.2771, "rew_std": 967.5778, "Agent": "PPO"}, {"env_step": 1628160, "rew": 3601.9068, "rew_std": 1035.0799, "Agent": "PPO"}, {"env_step": 1658880, "rew": 3446.1252, "rew_std": 820.5245, "Agent": "PPO"}, {"env_step": 1689600, "rew": 3505.6386, "rew_std": 968.1454, "Agent": "PPO"}, {"env_step": 1720320, "rew": 3687.4512, "rew_std": 867.7069, "Agent": "PPO"}, {"env_step": 1751040, "rew": 3615.7173, "rew_std": 839.4768, "Agent": "PPO"}, {"env_step": 1781760, "rew": 3712.6804, "rew_std": 957.1922, "Agent": "PPO"}, {"env_step": 1812480, "rew": 3525.5926, "rew_std": 825.2101, "Agent": "PPO"}, {"env_step": 1843200, "rew": 3679.4306, "rew_std": 753.9294, "Agent": "PPO"}, {"env_step": 1873920, "rew": 3758.6323, "rew_std": 770.2905, "Agent": "PPO"}, {"env_step": 1904640, "rew": 3481.2124, "rew_std": 756.4533, "Agent": "PPO"}, {"env_step": 1935360, "rew": 3420.7446, "rew_std": 805.4886, "Agent": "PPO"}, {"env_step": 1966080, "rew": 3494.1347, "rew_std": 752.0038, "Agent": "PPO"}, {"env_step": 1996800, "rew": 3577.4866, "rew_std": 339.4057, "Agent": "PPO"}, {"env_step": 2027520, "rew": 3639.4031, "rew_std": 971.0206, "Agent": "PPO"}, {"env_step": 2058240, "rew": 3519.3977, "rew_std": 713.6278, "Agent": "PPO"}, {"env_step": 2088960, "rew": 3512.7617, "rew_std": 780.0977, "Agent": "PPO"}, {"env_step": 2119680, "rew": 3530.8447, "rew_std": 636.9476, "Agent": "PPO"}, {"env_step": 2150400, "rew": 3523.7627, "rew_std": 741.5985, "Agent": "PPO"}, {"env_step": 2181120, "rew": 3564.3577, "rew_std": 708.5225, "Agent": "PPO"}, {"env_step": 2211840, "rew": 3576.1605, "rew_std": 866.6277, "Agent": "PPO"}, {"env_step": 2242560, "rew": 3804.0978, "rew_std": 705.8611, "Agent": "PPO"}, {"env_step": 2273280, "rew": 3599.313, "rew_std": 699.1721, "Agent": "PPO"}, {"env_step": 2304000, "rew": 3721.4555, "rew_std": 711.6818, "Agent": "PPO"}, {"env_step": 2334720, "rew": 3532.7069, "rew_std": 837.1282, "Agent": "PPO"}, {"env_step": 2365440, "rew": 3608.8838, "rew_std": 1090.2108, "Agent": "PPO"}, {"env_step": 2396160, "rew": 3910.161, "rew_std": 678.6507, "Agent": "PPO"}, {"env_step": 2426880, "rew": 3426.0564, "rew_std": 1185.8051, "Agent": "PPO"}, {"env_step": 2457600, "rew": 3679.0407, "rew_std": 873.2651, "Agent": "PPO"}, {"env_step": 2488320, "rew": 3757.3899, "rew_std": 950.9568, "Agent": "PPO"}, {"env_step": 2519040, "rew": 3813.3742, "rew_std": 764.5632, "Agent": "PPO"}, {"env_step": 2549760, "rew": 3773.2322, "rew_std": 727.4568, "Agent": "PPO"}, {"env_step": 2580480, "rew": 3639.3853, "rew_std": 827.8082, "Agent": "PPO"}, {"env_step": 2611200, "rew": 3704.148, "rew_std": 691.7173, "Agent": "PPO"}, {"env_step": 2641920, "rew": 3798.3913, "rew_std": 694.0727, "Agent": "PPO"}, {"env_step": 2672640, "rew": 3826.2185, "rew_std": 891.9137, "Agent": "PPO"}, {"env_step": 2703360, "rew": 3965.214, "rew_std": 820.4513, "Agent": "PPO"}, {"env_step": 2734080, "rew": 3644.2885, "rew_std": 977.7322, "Agent": "PPO"}, {"env_step": 2764800, "rew": 3832.7547, "rew_std": 857.0111, "Agent": "PPO"}, {"env_step": 2795520, "rew": 3870.129, "rew_std": 816.0748, "Agent": "PPO"}, {"env_step": 2826240, "rew": 3877.0556, "rew_std": 882.7163, "Agent": "PPO"}, {"env_step": 2856960, "rew": 4003.1883, "rew_std": 676.3348, "Agent": "PPO"}, {"env_step": 2887680, "rew": 3668.3481, "rew_std": 838.448, "Agent": "PPO"}, {"env_step": 2918400, "rew": 4079.2655, "rew_std": 880.1724, "Agent": "PPO"}, {"env_step": 2949120, "rew": 3730.9786, "rew_std": 834.9895, "Agent": "PPO"}, {"env_step": 2979840, "rew": 3877.0596, "rew_std": 878.0078, "Agent": "PPO"}, {"env_step": 3010560, "rew": 3750.2836, "rew_std": 849.4658, "Agent": "PPO"}, {"env_step": 3041280, "rew": 4148.1549, "rew_std": 852.399, "Agent": "PPO"}, {"env_step": 3072000, "rew": 3803.2651, "rew_std": 890.7726, "Agent": "PPO"}, {"env_step": 0, "rew": 27.7176, "rew_std": 10.7358, "Agent": "REINFORCE"}, {"env_step": 30720, "rew": 33.3533, "rew_std": 15.3157, "Agent": "REINFORCE"}, {"env_step": 61440, "rew": 41.4711, "rew_std": 14.8108, "Agent": "REINFORCE"}, {"env_step": 92160, "rew": 48.6603, "rew_std": 12.8797, "Agent": "REINFORCE"}, {"env_step": 122880, "rew": 36.7399, "rew_std": 16.7203, "Agent": "REINFORCE"}, {"env_step": 153600, "rew": 44.7029, "rew_std": 12.1446, "Agent": "REINFORCE"}, {"env_step": 184320, "rew": 42.3812, "rew_std": 13.0596, "Agent": "REINFORCE"}, {"env_step": 215040, "rew": 68.8463, "rew_std": 20.4335, "Agent": "REINFORCE"}, {"env_step": 245760, "rew": 41.8285, "rew_std": 18.668, "Agent": "REINFORCE"}, {"env_step": 276480, "rew": 61.6224, "rew_std": 13.3752, "Agent": "REINFORCE"}, {"env_step": 307200, "rew": 58.2795, "rew_std": 22.9679, "Agent": "REINFORCE"}, {"env_step": 337920, "rew": 69.899, "rew_std": 24.5287, "Agent": "REINFORCE"}, {"env_step": 368640, "rew": 71.4402, "rew_std": 22.2039, "Agent": "REINFORCE"}, {"env_step": 399360, "rew": 64.7685, "rew_std": 27.8313, "Agent": "REINFORCE"}, {"env_step": 430080, "rew": 77.4212, "rew_std": 20.5991, "Agent": "REINFORCE"}, {"env_step": 460800, "rew": 81.0816, "rew_std": 32.3568, "Agent": "REINFORCE"}, {"env_step": 491520, "rew": 100.2472, "rew_std": 29.5934, "Agent": "REINFORCE"}, {"env_step": 522240, "rew": 105.3082, "rew_std": 36.861, "Agent": "REINFORCE"}, {"env_step": 552960, "rew": 88.4532, "rew_std": 24.8457, "Agent": "REINFORCE"}, {"env_step": 583680, "rew": 132.3005, "rew_std": 36.845, "Agent": "REINFORCE"}, {"env_step": 614400, "rew": 115.8723, "rew_std": 45.9245, "Agent": "REINFORCE"}, {"env_step": 645120, "rew": 128.5706, "rew_std": 31.6052, "Agent": "REINFORCE"}, {"env_step": 675840, "rew": 139.1624, "rew_std": 38.3564, "Agent": "REINFORCE"}, {"env_step": 706560, "rew": 122.3089, "rew_std": 22.9594, "Agent": "REINFORCE"}, {"env_step": 737280, "rew": 139.8168, "rew_std": 36.04, "Agent": "REINFORCE"}, {"env_step": 768000, "rew": 135.3548, "rew_std": 31.8188, "Agent": "REINFORCE"}, {"env_step": 798720, "rew": 154.814, "rew_std": 36.1231, "Agent": "REINFORCE"}, {"env_step": 829440, "rew": 138.0523, "rew_std": 23.4258, "Agent": "REINFORCE"}, {"env_step": 860160, "rew": 154.9802, "rew_std": 28.0355, "Agent": "REINFORCE"}, {"env_step": 890880, "rew": 163.9125, "rew_std": 46.5653, "Agent": "REINFORCE"}, {"env_step": 921600, "rew": 178.5543, "rew_std": 39.1556, "Agent": "REINFORCE"}, {"env_step": 952320, "rew": 148.4363, "rew_std": 31.9275, "Agent": "REINFORCE"}, {"env_step": 983040, "rew": 173.0948, "rew_std": 35.8674, "Agent": "REINFORCE"}, {"env_step": 1013760, "rew": 184.1079, "rew_std": 48.525, "Agent": "REINFORCE"}, {"env_step": 1044480, "rew": 188.7721, "rew_std": 47.7577, "Agent": "REINFORCE"}, {"env_step": 1075200, "rew": 222.5404, "rew_std": 45.0577, "Agent": "REINFORCE"}, {"env_step": 1105920, "rew": 204.1252, "rew_std": 58.2783, "Agent": "REINFORCE"}, {"env_step": 1136640, "rew": 227.1378, "rew_std": 68.7146, "Agent": "REINFORCE"}, {"env_step": 1167360, "rew": 205.754, "rew_std": 21.5251, "Agent": "REINFORCE"}, {"env_step": 1198080, "rew": 195.8821, "rew_std": 38.2101, "Agent": "REINFORCE"}, {"env_step": 1228800, "rew": 222.6474, "rew_std": 26.4, "Agent": "REINFORCE"}, {"env_step": 1259520, "rew": 215.2108, "rew_std": 46.4782, "Agent": "REINFORCE"}, {"env_step": 1290240, "rew": 225.4944, "rew_std": 47.4316, "Agent": "REINFORCE"}, {"env_step": 1320960, "rew": 247.2296, "rew_std": 41.7104, "Agent": "REINFORCE"}, {"env_step": 1351680, "rew": 248.6715, "rew_std": 42.0487, "Agent": "REINFORCE"}, {"env_step": 1382400, "rew": 250.6153, "rew_std": 48.6851, "Agent": "REINFORCE"}, {"env_step": 1413120, "rew": 255.4334, "rew_std": 47.6847, "Agent": "REINFORCE"}, {"env_step": 1443840, "rew": 260.1298, "rew_std": 50.7508, "Agent": "REINFORCE"}, {"env_step": 1474560, "rew": 285.7685, "rew_std": 47.8319, "Agent": "REINFORCE"}, {"env_step": 1505280, "rew": 303.9189, "rew_std": 58.4329, "Agent": "REINFORCE"}, {"env_step": 1536000, "rew": 299.0006, "rew_std": 55.5217, "Agent": "REINFORCE"}, {"env_step": 1566720, "rew": 291.9584, "rew_std": 48.0131, "Agent": "REINFORCE"}, {"env_step": 1597440, "rew": 298.1129, "rew_std": 71.0903, "Agent": "REINFORCE"}, {"env_step": 1628160, "rew": 295.2096, "rew_std": 55.1375, "Agent": "REINFORCE"}, {"env_step": 1658880, "rew": 256.9214, "rew_std": 31.8228, "Agent": "REINFORCE"}, {"env_step": 1689600, "rew": 306.4976, "rew_std": 55.3333, "Agent": "REINFORCE"}, {"env_step": 1720320, "rew": 309.0145, "rew_std": 48.844, "Agent": "REINFORCE"}, {"env_step": 1751040, "rew": 292.7145, "rew_std": 49.9569, "Agent": "REINFORCE"}, {"env_step": 1781760, "rew": 297.1903, "rew_std": 57.495, "Agent": "REINFORCE"}, {"env_step": 1812480, "rew": 285.1572, "rew_std": 77.7148, "Agent": "REINFORCE"}, {"env_step": 1843200, "rew": 302.7925, "rew_std": 98.092, "Agent": "REINFORCE"}, {"env_step": 1873920, "rew": 312.8332, "rew_std": 80.4849, "Agent": "REINFORCE"}, {"env_step": 1904640, "rew": 328.3923, "rew_std": 44.7151, "Agent": "REINFORCE"}, {"env_step": 1935360, "rew": 353.9316, "rew_std": 50.9284, "Agent": "REINFORCE"}, {"env_step": 1966080, "rew": 326.6882, "rew_std": 89.6125, "Agent": "REINFORCE"}, {"env_step": 1996800, "rew": 304.3854, "rew_std": 78.2457, "Agent": "REINFORCE"}, {"env_step": 2027520, "rew": 364.8491, "rew_std": 45.1922, "Agent": "REINFORCE"}, {"env_step": 2058240, "rew": 301.7265, "rew_std": 74.8352, "Agent": "REINFORCE"}, {"env_step": 2088960, "rew": 324.6792, "rew_std": 53.6144, "Agent": "REINFORCE"}, {"env_step": 2119680, "rew": 341.6978, "rew_std": 63.9434, "Agent": "REINFORCE"}, {"env_step": 2150400, "rew": 363.9016, "rew_std": 64.7186, "Agent": "REINFORCE"}, {"env_step": 2181120, "rew": 374.0682, "rew_std": 45.6372, "Agent": "REINFORCE"}, {"env_step": 2211840, "rew": 385.5681, "rew_std": 89.3322, "Agent": "REINFORCE"}, {"env_step": 2242560, "rew": 378.1436, "rew_std": 86.5271, "Agent": "REINFORCE"}, {"env_step": 2273280, "rew": 335.9349, "rew_std": 75.4024, "Agent": "REINFORCE"}, {"env_step": 2304000, "rew": 423.9954, "rew_std": 65.6686, "Agent": "REINFORCE"}, {"env_step": 2334720, "rew": 385.9904, "rew_std": 49.5371, "Agent": "REINFORCE"}, {"env_step": 2365440, "rew": 373.246, "rew_std": 87.2209, "Agent": "REINFORCE"}, {"env_step": 2396160, "rew": 360.8469, "rew_std": 79.2568, "Agent": "REINFORCE"}, {"env_step": 2426880, "rew": 417.3086, "rew_std": 99.0708, "Agent": "REINFORCE"}, {"env_step": 2457600, "rew": 385.6445, "rew_std": 93.4971, "Agent": "REINFORCE"}, {"env_step": 2488320, "rew": 416.7968, "rew_std": 61.8736, "Agent": "REINFORCE"}, {"env_step": 2519040, "rew": 380.9239, "rew_std": 72.2998, "Agent": "REINFORCE"}, {"env_step": 2549760, "rew": 356.601, "rew_std": 79.8959, "Agent": "REINFORCE"}, {"env_step": 2580480, "rew": 437.0575, "rew_std": 77.9993, "Agent": "REINFORCE"}, {"env_step": 2611200, "rew": 393.3934, "rew_std": 82.0512, "Agent": "REINFORCE"}, {"env_step": 2641920, "rew": 390.14, "rew_std": 74.0242, "Agent": "REINFORCE"}, {"env_step": 2672640, "rew": 380.3886, "rew_std": 44.6099, "Agent": "REINFORCE"}, {"env_step": 2703360, "rew": 411.4326, "rew_std": 39.9505, "Agent": "REINFORCE"}, {"env_step": 2734080, "rew": 426.8174, "rew_std": 43.9964, "Agent": "REINFORCE"}, {"env_step": 2764800, "rew": 390.6643, "rew_std": 66.2562, "Agent": "REINFORCE"}, {"env_step": 2795520, "rew": 396.97, "rew_std": 104.7072, "Agent": "REINFORCE"}, {"env_step": 2826240, "rew": 467.2725, "rew_std": 48.5015, "Agent": "REINFORCE"}, {"env_step": 2856960, "rew": 374.8978, "rew_std": 108.3885, "Agent": "REINFORCE"}, {"env_step": 2887680, "rew": 450.2795, "rew_std": 95.4089, "Agent": "REINFORCE"}, {"env_step": 2918400, "rew": 428.4842, "rew_std": 111.6204, "Agent": "REINFORCE"}, {"env_step": 2949120, "rew": 417.3782, "rew_std": 91.5549, "Agent": "REINFORCE"}, {"env_step": 2979840, "rew": 383.2412, "rew_std": 78.9286, "Agent": "REINFORCE"}, {"env_step": 3010560, "rew": 448.1727, "rew_std": 133.4732, "Agent": "REINFORCE"}, {"env_step": 3041280, "rew": 474.9232, "rew_std": 133.5394, "Agent": "REINFORCE"}, {"env_step": 3072000, "rew": 458.941, "rew_std": 87.7709, "Agent": "REINFORCE"}, {"env_step": 3102720, "rew": 434.3256, "rew_std": 119.6724, "Agent": "REINFORCE"}, {"env_step": 3133440, "rew": 425.1408, "rew_std": 113.6893, "Agent": "REINFORCE"}, {"env_step": 3164160, "rew": 445.6428, "rew_std": 89.4675, "Agent": "REINFORCE"}, {"env_step": 3194880, "rew": 441.357, "rew_std": 89.8821, "Agent": "REINFORCE"}, {"env_step": 3225600, "rew": 469.4754, "rew_std": 94.0679, "Agent": "REINFORCE"}, {"env_step": 3256320, "rew": 433.924, "rew_std": 121.3369, "Agent": "REINFORCE"}, {"env_step": 3287040, "rew": 393.9809, "rew_std": 122.1391, "Agent": "REINFORCE"}, {"env_step": 3317760, "rew": 442.0835, "rew_std": 97.7841, "Agent": "REINFORCE"}, {"env_step": 3348480, "rew": 433.7272, "rew_std": 61.3641, "Agent": "REINFORCE"}, {"env_step": 3379200, "rew": 447.5686, "rew_std": 84.2138, "Agent": "REINFORCE"}, {"env_step": 3409920, "rew": 497.7688, "rew_std": 86.3697, "Agent": "REINFORCE"}, {"env_step": 3440640, "rew": 533.3218, "rew_std": 71.5671, "Agent": "REINFORCE"}, {"env_step": 3471360, "rew": 439.6428, "rew_std": 62.7993, "Agent": "REINFORCE"}, {"env_step": 3502080, "rew": 459.9121, "rew_std": 108.291, "Agent": "REINFORCE"}, {"env_step": 3532800, "rew": 459.4379, "rew_std": 132.8943, "Agent": "REINFORCE"}, {"env_step": 3563520, "rew": 470.9714, "rew_std": 106.7905, "Agent": "REINFORCE"}, {"env_step": 3594240, "rew": 502.5269, "rew_std": 112.8716, "Agent": "REINFORCE"}, {"env_step": 3624960, "rew": 518.7053, "rew_std": 85.9213, "Agent": "REINFORCE"}, {"env_step": 3655680, "rew": 510.9007, "rew_std": 81.0547, "Agent": "REINFORCE"}, {"env_step": 3686400, "rew": 422.44, "rew_std": 101.416, "Agent": "REINFORCE"}, {"env_step": 3717120, "rew": 509.2242, "rew_std": 133.3777, "Agent": "REINFORCE"}, {"env_step": 3747840, "rew": 478.1535, "rew_std": 142.6283, "Agent": "REINFORCE"}, {"env_step": 3778560, "rew": 424.1849, "rew_std": 55.8675, "Agent": "REINFORCE"}, {"env_step": 3809280, "rew": 414.8215, "rew_std": 103.8898, "Agent": "REINFORCE"}, {"env_step": 3840000, "rew": 440.345, "rew_std": 167.3447, "Agent": "REINFORCE"}, {"env_step": 3870720, "rew": 476.521, "rew_std": 131.104, "Agent": "REINFORCE"}, {"env_step": 3901440, "rew": 519.2961, "rew_std": 98.1275, "Agent": "REINFORCE"}, {"env_step": 3932160, "rew": 560.9757, "rew_std": 149.2555, "Agent": "REINFORCE"}, {"env_step": 3962880, "rew": 468.661, "rew_std": 148.3693, "Agent": "REINFORCE"}, {"env_step": 3993600, "rew": 429.7429, "rew_std": 92.4378, "Agent": "REINFORCE"}, {"env_step": 4024320, "rew": 530.7159, "rew_std": 115.3881, "Agent": "REINFORCE"}, {"env_step": 4055040, "rew": 504.7885, "rew_std": 139.296, "Agent": "REINFORCE"}, {"env_step": 4085760, "rew": 527.264, "rew_std": 114.0875, "Agent": "REINFORCE"}, {"env_step": 4116480, "rew": 524.9679, "rew_std": 97.0679, "Agent": "REINFORCE"}, {"env_step": 4147200, "rew": 502.5799, "rew_std": 51.4227, "Agent": "REINFORCE"}, {"env_step": 4177920, "rew": 558.6022, "rew_std": 141.0198, "Agent": "REINFORCE"}, {"env_step": 4208640, "rew": 542.5564, "rew_std": 143.6826, "Agent": "REINFORCE"}, {"env_step": 4239360, "rew": 565.7178, "rew_std": 123.4404, "Agent": "REINFORCE"}, {"env_step": 4270080, "rew": 588.5188, "rew_std": 91.6356, "Agent": "REINFORCE"}, {"env_step": 4300800, "rew": 509.4929, "rew_std": 88.4461, "Agent": "REINFORCE"}, {"env_step": 4331520, "rew": 560.255, "rew_std": 110.7314, "Agent": "REINFORCE"}, {"env_step": 4362240, "rew": 584.3804, "rew_std": 115.0177, "Agent": "REINFORCE"}, {"env_step": 4392960, "rew": 511.6448, "rew_std": 115.7761, "Agent": "REINFORCE"}, {"env_step": 4423680, "rew": 548.2895, "rew_std": 97.5672, "Agent": "REINFORCE"}, {"env_step": 4454400, "rew": 643.0676, "rew_std": 147.6213, "Agent": "REINFORCE"}, {"env_step": 4485120, "rew": 549.4917, "rew_std": 92.6585, "Agent": "REINFORCE"}, {"env_step": 4515840, "rew": 514.8249, "rew_std": 137.2659, "Agent": "REINFORCE"}, {"env_step": 4546560, "rew": 584.2434, "rew_std": 127.4338, "Agent": "REINFORCE"}, {"env_step": 4577280, "rew": 603.557, "rew_std": 106.7744, "Agent": "REINFORCE"}, {"env_step": 4608000, "rew": 509.8114, "rew_std": 113.999, "Agent": "REINFORCE"}, {"env_step": 4638720, "rew": 570.2314, "rew_std": 128.3169, "Agent": "REINFORCE"}, {"env_step": 4669440, "rew": 561.8519, "rew_std": 167.7081, "Agent": "REINFORCE"}, {"env_step": 4700160, "rew": 512.5841, "rew_std": 105.8006, "Agent": "REINFORCE"}, {"env_step": 4730880, "rew": 579.3501, "rew_std": 131.1142, "Agent": "REINFORCE"}, {"env_step": 4761600, "rew": 580.1297, "rew_std": 136.0232, "Agent": "REINFORCE"}, {"env_step": 4792320, "rew": 515.8023, "rew_std": 102.5996, "Agent": "REINFORCE"}, {"env_step": 4823040, "rew": 598.8872, "rew_std": 158.7483, "Agent": "REINFORCE"}, {"env_step": 4853760, "rew": 622.3687, "rew_std": 95.173, "Agent": "REINFORCE"}, {"env_step": 4884480, "rew": 544.3778, "rew_std": 112.6735, "Agent": "REINFORCE"}, {"env_step": 4915200, "rew": 588.5974, "rew_std": 113.4368, "Agent": "REINFORCE"}, {"env_step": 4945920, "rew": 550.9882, "rew_std": 133.2863, "Agent": "REINFORCE"}, {"env_step": 4976640, "rew": 541.7536, "rew_std": 89.7816, "Agent": "REINFORCE"}, {"env_step": 5007360, "rew": 603.986, "rew_std": 135.1224, "Agent": "REINFORCE"}, {"env_step": 5038080, "rew": 583.561, "rew_std": 114.0059, "Agent": "REINFORCE"}, {"env_step": 5068800, "rew": 567.4875, "rew_std": 147.1721, "Agent": "REINFORCE"}, {"env_step": 5099520, "rew": 572.4557, "rew_std": 156.9581, "Agent": "REINFORCE"}, {"env_step": 5130240, "rew": 666.3246, "rew_std": 176.1906, "Agent": "REINFORCE"}, {"env_step": 5160960, "rew": 597.1891, "rew_std": 151.6925, "Agent": "REINFORCE"}, {"env_step": 5191680, "rew": 597.5578, "rew_std": 165.4627, "Agent": "REINFORCE"}, {"env_step": 5222400, "rew": 559.7764, "rew_std": 121.6349, "Agent": "REINFORCE"}, {"env_step": 5253120, "rew": 614.2217, "rew_std": 134.1647, "Agent": "REINFORCE"}, {"env_step": 5283840, "rew": 535.2713, "rew_std": 184.4832, "Agent": "REINFORCE"}, {"env_step": 5314560, "rew": 644.3689, "rew_std": 122.9989, "Agent": "REINFORCE"}, {"env_step": 5345280, "rew": 576.1621, "rew_std": 149.0917, "Agent": "REINFORCE"}, {"env_step": 5376000, "rew": 561.303, "rew_std": 165.4708, "Agent": "REINFORCE"}, {"env_step": 5406720, "rew": 648.7732, "rew_std": 219.9609, "Agent": "REINFORCE"}, {"env_step": 5437440, "rew": 576.5953, "rew_std": 140.9941, "Agent": "REINFORCE"}, {"env_step": 5468160, "rew": 562.7847, "rew_std": 152.2883, "Agent": "REINFORCE"}, {"env_step": 5498880, "rew": 655.4698, "rew_std": 101.2357, "Agent": "REINFORCE"}, {"env_step": 5529600, "rew": 626.2532, "rew_std": 225.647, "Agent": "REINFORCE"}, {"env_step": 5560320, "rew": 561.6642, "rew_std": 162.0297, "Agent": "REINFORCE"}, {"env_step": 5591040, "rew": 621.637, "rew_std": 144.4238, "Agent": "REINFORCE"}, {"env_step": 5621760, "rew": 621.095, "rew_std": 299.2659, "Agent": "REINFORCE"}, {"env_step": 5652480, "rew": 591.8279, "rew_std": 160.6478, "Agent": "REINFORCE"}, {"env_step": 5683200, "rew": 566.1898, "rew_std": 124.7465, "Agent": "REINFORCE"}, {"env_step": 5713920, "rew": 571.6892, "rew_std": 99.2399, "Agent": "REINFORCE"}, {"env_step": 5744640, "rew": 518.5974, "rew_std": 110.1643, "Agent": "REINFORCE"}, {"env_step": 5775360, "rew": 507.0606, "rew_std": 118.8786, "Agent": "REINFORCE"}, {"env_step": 5806080, "rew": 645.1324, "rew_std": 118.7828, "Agent": "REINFORCE"}, {"env_step": 5836800, "rew": 575.8455, "rew_std": 130.6544, "Agent": "REINFORCE"}, {"env_step": 5867520, "rew": 580.2341, "rew_std": 148.9996, "Agent": "REINFORCE"}, {"env_step": 5898240, "rew": 639.1779, "rew_std": 125.3437, "Agent": "REINFORCE"}, {"env_step": 5928960, "rew": 576.9596, "rew_std": 124.1505, "Agent": "REINFORCE"}, {"env_step": 5959680, "rew": 644.9708, "rew_std": 66.2187, "Agent": "REINFORCE"}, {"env_step": 5990400, "rew": 694.9196, "rew_std": 177.9414, "Agent": "REINFORCE"}, {"env_step": 6021120, "rew": 694.7615, "rew_std": 220.8509, "Agent": "REINFORCE"}, {"env_step": 6051840, "rew": 729.0638, "rew_std": 225.7093, "Agent": "REINFORCE"}, {"env_step": 6082560, "rew": 664.6531, "rew_std": 165.6677, "Agent": "REINFORCE"}, {"env_step": 6113280, "rew": 637.6006, "rew_std": 218.98, "Agent": "REINFORCE"}, {"env_step": 6144000, "rew": 528.8907, "rew_std": 108.5123, "Agent": "REINFORCE"}, {"env_step": 6174720, "rew": 582.7341, "rew_std": 142.043, "Agent": "REINFORCE"}, {"env_step": 6205440, "rew": 655.7331, "rew_std": 141.1264, "Agent": "REINFORCE"}, {"env_step": 6236160, "rew": 673.7024, "rew_std": 233.1061, "Agent": "REINFORCE"}, {"env_step": 6266880, "rew": 635.9546, "rew_std": 217.1002, "Agent": "REINFORCE"}, {"env_step": 6297600, "rew": 666.5248, "rew_std": 209.9766, "Agent": "REINFORCE"}, {"env_step": 6328320, "rew": 680.161, "rew_std": 140.829, "Agent": "REINFORCE"}, {"env_step": 6359040, "rew": 685.1298, "rew_std": 129.4409, "Agent": "REINFORCE"}, {"env_step": 6389760, "rew": 744.9942, "rew_std": 400.4335, "Agent": "REINFORCE"}, {"env_step": 6420480, "rew": 673.2758, "rew_std": 157.3355, "Agent": "REINFORCE"}, {"env_step": 6451200, "rew": 735.2884, "rew_std": 171.1652, "Agent": "REINFORCE"}, {"env_step": 6481920, "rew": 788.1765, "rew_std": 239.2203, "Agent": "REINFORCE"}, {"env_step": 6512640, "rew": 627.7094, "rew_std": 209.5059, "Agent": "REINFORCE"}, {"env_step": 6543360, "rew": 702.6232, "rew_std": 151.2339, "Agent": "REINFORCE"}, {"env_step": 6574080, "rew": 761.3974, "rew_std": 214.923, "Agent": "REINFORCE"}, {"env_step": 6604800, "rew": 715.0448, "rew_std": 193.3326, "Agent": "REINFORCE"}, {"env_step": 6635520, "rew": 687.1525, "rew_std": 80.5748, "Agent": "REINFORCE"}, {"env_step": 6666240, "rew": 738.8947, "rew_std": 197.2139, "Agent": "REINFORCE"}, {"env_step": 6696960, "rew": 735.0513, "rew_std": 243.043, "Agent": "REINFORCE"}, {"env_step": 6727680, "rew": 661.899, "rew_std": 137.0016, "Agent": "REINFORCE"}, {"env_step": 6758400, "rew": 717.0317, "rew_std": 115.1148, "Agent": "REINFORCE"}, {"env_step": 6789120, "rew": 713.7875, "rew_std": 190.9573, "Agent": "REINFORCE"}, {"env_step": 6819840, "rew": 753.5813, "rew_std": 224.2372, "Agent": "REINFORCE"}, {"env_step": 6850560, "rew": 827.5102, "rew_std": 200.4547, "Agent": "REINFORCE"}, {"env_step": 6881280, "rew": 790.6831, "rew_std": 281.7583, "Agent": "REINFORCE"}, {"env_step": 6912000, "rew": 828.0825, "rew_std": 186.1612, "Agent": "REINFORCE"}, {"env_step": 6942720, "rew": 740.5346, "rew_std": 192.5644, "Agent": "REINFORCE"}, {"env_step": 6973440, "rew": 768.3898, "rew_std": 173.439, "Agent": "REINFORCE"}, {"env_step": 7004160, "rew": 767.4472, "rew_std": 81.7889, "Agent": "REINFORCE"}, {"env_step": 7034880, "rew": 843.0486, "rew_std": 207.4263, "Agent": "REINFORCE"}, {"env_step": 7065600, "rew": 917.0862, "rew_std": 175.6432, "Agent": "REINFORCE"}, {"env_step": 7096320, "rew": 764.7748, "rew_std": 186.3004, "Agent": "REINFORCE"}, {"env_step": 7127040, "rew": 818.4783, "rew_std": 227.9405, "Agent": "REINFORCE"}, {"env_step": 7157760, "rew": 840.2352, "rew_std": 254.5002, "Agent": "REINFORCE"}, {"env_step": 7188480, "rew": 731.5163, "rew_std": 233.0287, "Agent": "REINFORCE"}, {"env_step": 7219200, "rew": 804.892, "rew_std": 206.5893, "Agent": "REINFORCE"}, {"env_step": 7249920, "rew": 721.1122, "rew_std": 119.3359, "Agent": "REINFORCE"}, {"env_step": 7280640, "rew": 745.0533, "rew_std": 216.2748, "Agent": "REINFORCE"}, {"env_step": 7311360, "rew": 789.4155, "rew_std": 201.4028, "Agent": "REINFORCE"}, {"env_step": 7342080, "rew": 778.7515, "rew_std": 169.0804, "Agent": "REINFORCE"}, {"env_step": 7372800, "rew": 722.2184, "rew_std": 159.7289, "Agent": "REINFORCE"}, {"env_step": 7403520, "rew": 730.5377, "rew_std": 244.0747, "Agent": "REINFORCE"}, {"env_step": 7434240, "rew": 821.842, "rew_std": 228.4728, "Agent": "REINFORCE"}, {"env_step": 7464960, "rew": 773.8623, "rew_std": 236.9126, "Agent": "REINFORCE"}, {"env_step": 7495680, "rew": 785.3531, "rew_std": 206.1696, "Agent": "REINFORCE"}, {"env_step": 7526400, "rew": 851.5261, "rew_std": 228.7796, "Agent": "REINFORCE"}, {"env_step": 7557120, "rew": 831.5571, "rew_std": 212.207, "Agent": "REINFORCE"}, {"env_step": 7587840, "rew": 813.1027, "rew_std": 244.633, "Agent": "REINFORCE"}, {"env_step": 7618560, "rew": 924.0815, "rew_std": 257.4552, "Agent": "REINFORCE"}, {"env_step": 7649280, "rew": 849.9347, "rew_std": 237.2322, "Agent": "REINFORCE"}, {"env_step": 7680000, "rew": 809.7352, "rew_std": 161.8209, "Agent": "REINFORCE"}, {"env_step": 7710720, "rew": 899.7744, "rew_std": 126.8619, "Agent": "REINFORCE"}, {"env_step": 7741440, "rew": 775.6868, "rew_std": 111.1748, "Agent": "REINFORCE"}, {"env_step": 7772160, "rew": 783.3401, "rew_std": 172.1061, "Agent": "REINFORCE"}, {"env_step": 7802880, "rew": 775.995, "rew_std": 238.964, "Agent": "REINFORCE"}, {"env_step": 7833600, "rew": 738.9181, "rew_std": 247.9718, "Agent": "REINFORCE"}, {"env_step": 7864320, "rew": 845.1103, "rew_std": 144.6614, "Agent": "REINFORCE"}, {"env_step": 7895040, "rew": 846.7213, "rew_std": 251.5901, "Agent": "REINFORCE"}, {"env_step": 7925760, "rew": 903.9714, "rew_std": 248.4302, "Agent": "REINFORCE"}, {"env_step": 7956480, "rew": 885.5948, "rew_std": 214.2679, "Agent": "REINFORCE"}, {"env_step": 7987200, "rew": 797.1632, "rew_std": 213.6274, "Agent": "REINFORCE"}, {"env_step": 8017920, "rew": 750.0977, "rew_std": 243.3167, "Agent": "REINFORCE"}, {"env_step": 8048640, "rew": 775.8821, "rew_std": 222.1063, "Agent": "REINFORCE"}, {"env_step": 8079360, "rew": 776.964, "rew_std": 195.6586, "Agent": "REINFORCE"}, {"env_step": 8110080, "rew": 834.2439, "rew_std": 307.1783, "Agent": "REINFORCE"}, {"env_step": 8140800, "rew": 922.2053, "rew_std": 245.5225, "Agent": "REINFORCE"}, {"env_step": 8171520, "rew": 900.8791, "rew_std": 257.9531, "Agent": "REINFORCE"}, {"env_step": 8202240, "rew": 896.3298, "rew_std": 277.692, "Agent": "REINFORCE"}, {"env_step": 8232960, "rew": 915.0473, "rew_std": 384.9129, "Agent": "REINFORCE"}, {"env_step": 8263680, "rew": 830.1566, "rew_std": 127.4406, "Agent": "REINFORCE"}, {"env_step": 8294400, "rew": 889.0198, "rew_std": 312.3793, "Agent": "REINFORCE"}, {"env_step": 8325120, "rew": 888.688, "rew_std": 271.7225, "Agent": "REINFORCE"}, {"env_step": 8355840, "rew": 824.6942, "rew_std": 196.723, "Agent": "REINFORCE"}, {"env_step": 8386560, "rew": 863.9463, "rew_std": 227.5124, "Agent": "REINFORCE"}, {"env_step": 8417280, "rew": 870.4753, "rew_std": 236.4412, "Agent": "REINFORCE"}, {"env_step": 8448000, "rew": 818.8857, "rew_std": 222.3761, "Agent": "REINFORCE"}, {"env_step": 8478720, "rew": 776.188, "rew_std": 228.0678, "Agent": "REINFORCE"}, {"env_step": 8509440, "rew": 882.9349, "rew_std": 201.0272, "Agent": "REINFORCE"}, {"env_step": 8540160, "rew": 825.3203, "rew_std": 188.1268, "Agent": "REINFORCE"}, {"env_step": 8570880, "rew": 831.3546, "rew_std": 251.4183, "Agent": "REINFORCE"}, {"env_step": 8601600, "rew": 820.2887, "rew_std": 219.7999, "Agent": "REINFORCE"}, {"env_step": 8632320, "rew": 925.9424, "rew_std": 204.7964, "Agent": "REINFORCE"}, {"env_step": 8663040, "rew": 765.2604, "rew_std": 149.2658, "Agent": "REINFORCE"}, {"env_step": 8693760, "rew": 913.36, "rew_std": 347.7958, "Agent": "REINFORCE"}, {"env_step": 8724480, "rew": 747.734, "rew_std": 165.6218, "Agent": "REINFORCE"}, {"env_step": 8755200, "rew": 864.8618, "rew_std": 258.5876, "Agent": "REINFORCE"}, {"env_step": 8785920, "rew": 834.3716, "rew_std": 149.974, "Agent": "REINFORCE"}, {"env_step": 8816640, "rew": 942.7596, "rew_std": 318.8342, "Agent": "REINFORCE"}, {"env_step": 8847360, "rew": 819.8248, "rew_std": 136.2533, "Agent": "REINFORCE"}, {"env_step": 8878080, "rew": 939.1256, "rew_std": 345.884, "Agent": "REINFORCE"}, {"env_step": 8908800, "rew": 919.9905, "rew_std": 318.5415, "Agent": "REINFORCE"}, {"env_step": 8939520, "rew": 952.3609, "rew_std": 243.2247, "Agent": "REINFORCE"}, {"env_step": 8970240, "rew": 1002.9598, "rew_std": 411.5419, "Agent": "REINFORCE"}, {"env_step": 9000960, "rew": 978.2093, "rew_std": 150.1634, "Agent": "REINFORCE"}, {"env_step": 9031680, "rew": 1000.6892, "rew_std": 238.6836, "Agent": "REINFORCE"}, {"env_step": 9062400, "rew": 923.7965, "rew_std": 265.0906, "Agent": "REINFORCE"}, {"env_step": 9093120, "rew": 1027.5603, "rew_std": 356.688, "Agent": "REINFORCE"}, {"env_step": 9123840, "rew": 927.87, "rew_std": 218.0477, "Agent": "REINFORCE"}, {"env_step": 9154560, "rew": 947.8009, "rew_std": 196.2311, "Agent": "REINFORCE"}, {"env_step": 9185280, "rew": 1002.6192, "rew_std": 346.7466, "Agent": "REINFORCE"}, {"env_step": 9216000, "rew": 966.7356, "rew_std": 257.1045, "Agent": "REINFORCE"}, {"env_step": 9246720, "rew": 850.2131, "rew_std": 272.2633, "Agent": "REINFORCE"}, {"env_step": 9277440, "rew": 999.9818, "rew_std": 242.1809, "Agent": "REINFORCE"}, {"env_step": 9308160, "rew": 945.274, "rew_std": 292.8736, "Agent": "REINFORCE"}, {"env_step": 9338880, "rew": 1011.6188, "rew_std": 206.0314, "Agent": "REINFORCE"}, {"env_step": 9369600, "rew": 936.7706, "rew_std": 169.1316, "Agent": "REINFORCE"}, {"env_step": 9400320, "rew": 909.0751, "rew_std": 195.3988, "Agent": "REINFORCE"}, {"env_step": 9431040, "rew": 854.0486, "rew_std": 211.4514, "Agent": "REINFORCE"}, {"env_step": 9461760, "rew": 960.132, "rew_std": 259.6515, "Agent": "REINFORCE"}, {"env_step": 9492480, "rew": 907.91, "rew_std": 339.0151, "Agent": "REINFORCE"}, {"env_step": 9523200, "rew": 1108.1313, "rew_std": 323.0787, "Agent": "REINFORCE"}, {"env_step": 9553920, "rew": 1003.4416, "rew_std": 259.1606, "Agent": "REINFORCE"}, {"env_step": 9584640, "rew": 906.0153, "rew_std": 269.067, "Agent": "REINFORCE"}, {"env_step": 9615360, "rew": 832.5955, "rew_std": 223.1421, "Agent": "REINFORCE"}, {"env_step": 9646080, "rew": 870.3656, "rew_std": 172.9299, "Agent": "REINFORCE"}, {"env_step": 9676800, "rew": 952.6487, "rew_std": 256.4124, "Agent": "REINFORCE"}, {"env_step": 9707520, "rew": 891.1408, "rew_std": 258.5006, "Agent": "REINFORCE"}, {"env_step": 9738240, "rew": 1067.376, "rew_std": 426.0889, "Agent": "REINFORCE"}, {"env_step": 9768960, "rew": 994.8644, "rew_std": 235.152, "Agent": "REINFORCE"}, {"env_step": 9799680, "rew": 1036.7538, "rew_std": 350.4073, "Agent": "REINFORCE"}, {"env_step": 9830400, "rew": 941.5028, "rew_std": 290.7186, "Agent": "REINFORCE"}, {"env_step": 9861120, "rew": 1004.6263, "rew_std": 189.7126, "Agent": "REINFORCE"}, {"env_step": 9891840, "rew": 1090.99, "rew_std": 218.7887, "Agent": "REINFORCE"}, {"env_step": 9922560, "rew": 1025.0696, "rew_std": 270.5742, "Agent": "REINFORCE"}, {"env_step": 9953280, "rew": 1001.7405, "rew_std": 334.5234, "Agent": "REINFORCE"}, {"env_step": 9984000, "rew": 859.6256, "rew_std": 283.6046, "Agent": "REINFORCE"}, {"env_step": 10014720, "rew": 916.4069, "rew_std": 234.0294, "Agent": "REINFORCE"}, {"env_step": 10045440, "rew": 999.6534, "rew_std": 370.8359, "Agent": "REINFORCE"}, {"env_step": 10076160, "rew": 956.329, "rew_std": 303.8919, "Agent": "REINFORCE"}, {"env_step": 10106880, "rew": 966.2531, "rew_std": 375.6601, "Agent": "REINFORCE"}, {"env_step": 10137600, "rew": 940.921, "rew_std": 362.3317, "Agent": "REINFORCE"}, {"env_step": 10168320, "rew": 989.9863, "rew_std": 233.9977, "Agent": "REINFORCE"}, {"env_step": 10199040, "rew": 870.4539, "rew_std": 227.1541, "Agent": "REINFORCE"}, {"env_step": 10229760, "rew": 1087.806, "rew_std": 295.9252, "Agent": "REINFORCE"}, {"env_step": 0, "rew": -51.4935, "rew_std": 27.5503, "Agent": "A2C"}, {"env_step": 30000, "rew": -29.8644, "rew_std": 10.21, "Agent": "A2C"}, {"env_step": 60000, "rew": 8.1853, "rew_std": 18.2229, "Agent": "A2C"}, {"env_step": 90000, "rew": 72.4685, "rew_std": 23.1961, "Agent": "A2C"}, {"env_step": 120000, "rew": 166.4676, "rew_std": 47.3271, "Agent": "A2C"}, {"env_step": 150000, "rew": 295.9591, "rew_std": 53.7155, "Agent": "A2C"}, {"env_step": 180000, "rew": 433.0201, "rew_std": 61.09, "Agent": "A2C"}, {"env_step": 210000, "rew": 586.3078, "rew_std": 67.2288, "Agent": "A2C"}, {"env_step": 240000, "rew": 756.9108, "rew_std": 94.0155, "Agent": "A2C"}, {"env_step": 270000, "rew": 805.3976, "rew_std": 247.5444, "Agent": "A2C"}, {"env_step": 300000, "rew": 1061.0577, "rew_std": 248.3772, "Agent": "A2C"}, {"env_step": 330000, "rew": 1097.009, "rew_std": 287.8099, "Agent": "A2C"}, {"env_step": 360000, "rew": 1361.5714, "rew_std": 207.5136, "Agent": "A2C"}, {"env_step": 390000, "rew": 1461.0194, "rew_std": 208.5063, "Agent": "A2C"}, {"env_step": 420000, "rew": 1664.8355, "rew_std": 330.0613, "Agent": "A2C"}, {"env_step": 450000, "rew": 2027.2921, "rew_std": 406.546, "Agent": "A2C"}, {"env_step": 480000, "rew": 2167.3209, "rew_std": 296.5709, "Agent": "A2C"}, {"env_step": 510000, "rew": 2056.6103, "rew_std": 433.0343, "Agent": "A2C"}, {"env_step": 540000, "rew": 2277.6117, "rew_std": 415.327, "Agent": "A2C"}, {"env_step": 570000, "rew": 2426.6807, "rew_std": 406.55, "Agent": "A2C"}, {"env_step": 600000, "rew": 2246.863, "rew_std": 394.1466, "Agent": "A2C"}, {"env_step": 630000, "rew": 2542.0354, "rew_std": 308.5228, "Agent": "A2C"}, {"env_step": 660000, "rew": 2831.0715, "rew_std": 421.904, "Agent": "A2C"}, {"env_step": 690000, "rew": 2505.3813, "rew_std": 541.7428, "Agent": "A2C"}, {"env_step": 720000, "rew": 2858.1248, "rew_std": 808.4451, "Agent": "A2C"}, {"env_step": 750000, "rew": 2779.9051, "rew_std": 499.6664, "Agent": "A2C"}, {"env_step": 780000, "rew": 3057.9012, "rew_std": 438.8949, "Agent": "A2C"}, {"env_step": 810000, "rew": 2922.3404, "rew_std": 487.1237, "Agent": "A2C"}, {"env_step": 840000, "rew": 3062.2575, "rew_std": 570.3453, "Agent": "A2C"}, {"env_step": 870000, "rew": 3098.2249, "rew_std": 598.1617, "Agent": "A2C"}, {"env_step": 900000, "rew": 3468.0156, "rew_std": 356.8802, "Agent": "A2C"}, {"env_step": 930000, "rew": 3332.6813, "rew_std": 455.3087, "Agent": "A2C"}, {"env_step": 960000, "rew": 3485.3726, "rew_std": 433.075, "Agent": "A2C"}, {"env_step": 990000, "rew": 3532.2745, "rew_std": 355.396, "Agent": "A2C"}, {"env_step": 1020000, "rew": 3371.1942, "rew_std": 756.4559, "Agent": "A2C"}, {"env_step": 1050000, "rew": 3327.8508, "rew_std": 465.0158, "Agent": "A2C"}, {"env_step": 1080000, "rew": 3410.0672, "rew_std": 487.9373, "Agent": "A2C"}, {"env_step": 1110000, "rew": 3508.0405, "rew_std": 492.2738, "Agent": "A2C"}, {"env_step": 1140000, "rew": 3645.9659, "rew_std": 234.3579, "Agent": "A2C"}, {"env_step": 1170000, "rew": 3601.9476, "rew_std": 410.754, "Agent": "A2C"}, {"env_step": 1200000, "rew": 3697.1497, "rew_std": 336.5131, "Agent": "A2C"}, {"env_step": 1230000, "rew": 3600.163, "rew_std": 755.6142, "Agent": "A2C"}, {"env_step": 1260000, "rew": 3783.988, "rew_std": 365.581, "Agent": "A2C"}, {"env_step": 1290000, "rew": 3999.3058, "rew_std": 357.098, "Agent": "A2C"}, {"env_step": 1320000, "rew": 3882.6923, "rew_std": 427.2421, "Agent": "A2C"}, {"env_step": 1350000, "rew": 4203.9126, "rew_std": 322.069, "Agent": "A2C"}, {"env_step": 1380000, "rew": 4071.4676, "rew_std": 298.9797, "Agent": "A2C"}, {"env_step": 1410000, "rew": 3725.2422, "rew_std": 638.7074, "Agent": "A2C"}, {"env_step": 1440000, "rew": 3869.6779, "rew_std": 614.8018, "Agent": "A2C"}, {"env_step": 1470000, "rew": 3937.0898, "rew_std": 566.8286, "Agent": "A2C"}, {"env_step": 1500000, "rew": 4054.1049, "rew_std": 565.2622, "Agent": "A2C"}, {"env_step": 1530000, "rew": 4147.8125, "rew_std": 536.8616, "Agent": "A2C"}, {"env_step": 1560000, "rew": 4044.8505, "rew_std": 322.5931, "Agent": "A2C"}, {"env_step": 1590000, "rew": 4189.3446, "rew_std": 273.6783, "Agent": "A2C"}, {"env_step": 1620000, "rew": 4134.2459, "rew_std": 478.0475, "Agent": "A2C"}, {"env_step": 1650000, "rew": 4109.2237, "rew_std": 576.0219, "Agent": "A2C"}, {"env_step": 1680000, "rew": 4373.6784, "rew_std": 203.6434, "Agent": "A2C"}, {"env_step": 1710000, "rew": 4463.585, "rew_std": 379.672, "Agent": "A2C"}, {"env_step": 1740000, "rew": 4189.2454, "rew_std": 566.2247, "Agent": "A2C"}, {"env_step": 1770000, "rew": 4368.2683, "rew_std": 505.7004, "Agent": "A2C"}, {"env_step": 1800000, "rew": 4242.8701, "rew_std": 714.7195, "Agent": "A2C"}, {"env_step": 1830000, "rew": 4408.0621, "rew_std": 300.5416, "Agent": "A2C"}, {"env_step": 1860000, "rew": 4574.196, "rew_std": 279.109, "Agent": "A2C"}, {"env_step": 1890000, "rew": 4396.9066, "rew_std": 490.7734, "Agent": "A2C"}, {"env_step": 1920000, "rew": 4523.7742, "rew_std": 397.95, "Agent": "A2C"}, {"env_step": 1950000, "rew": 4554.3086, "rew_std": 592.726, "Agent": "A2C"}, {"env_step": 1980000, "rew": 4305.6687, "rew_std": 562.9579, "Agent": "A2C"}, {"env_step": 2010000, "rew": 4894.4541, "rew_std": 219.3345, "Agent": "A2C"}, {"env_step": 2040000, "rew": 4487.1695, "rew_std": 301.3349, "Agent": "A2C"}, {"env_step": 2070000, "rew": 4466.5832, "rew_std": 464.2145, "Agent": "A2C"}, {"env_step": 2100000, "rew": 4666.3744, "rew_std": 370.7685, "Agent": "A2C"}, {"env_step": 2130000, "rew": 4706.4277, "rew_std": 403.2524, "Agent": "A2C"}, {"env_step": 2160000, "rew": 4530.5346, "rew_std": 514.5, "Agent": "A2C"}, {"env_step": 2190000, "rew": 4873.5241, "rew_std": 324.0447, "Agent": "A2C"}, {"env_step": 2220000, "rew": 4810.5365, "rew_std": 344.5598, "Agent": "A2C"}, {"env_step": 2250000, "rew": 4721.3229, "rew_std": 238.1361, "Agent": "A2C"}, {"env_step": 2280000, "rew": 4736.7355, "rew_std": 413.251, "Agent": "A2C"}, {"env_step": 2310000, "rew": 4755.9058, "rew_std": 473.5718, "Agent": "A2C"}, {"env_step": 2340000, "rew": 4974.8084, "rew_std": 217.7102, "Agent": "A2C"}, {"env_step": 2370000, "rew": 5030.6218, "rew_std": 207.6396, "Agent": "A2C"}, {"env_step": 2400000, "rew": 4690.2767, "rew_std": 409.2839, "Agent": "A2C"}, {"env_step": 2430000, "rew": 5039.456, "rew_std": 318.2659, "Agent": "A2C"}, {"env_step": 2460000, "rew": 4773.1173, "rew_std": 416.1896, "Agent": "A2C"}, {"env_step": 2490000, "rew": 4789.5868, "rew_std": 502.159, "Agent": "A2C"}, {"env_step": 2520000, "rew": 4823.6652, "rew_std": 396.6441, "Agent": "A2C"}, {"env_step": 2550000, "rew": 4879.6176, "rew_std": 429.7661, "Agent": "A2C"}, {"env_step": 2580000, "rew": 4993.1127, "rew_std": 328.2962, "Agent": "A2C"}, {"env_step": 2610000, "rew": 4898.4844, "rew_std": 489.4285, "Agent": "A2C"}, {"env_step": 2640000, "rew": 5096.5163, "rew_std": 294.0463, "Agent": "A2C"}, {"env_step": 2670000, "rew": 4900.6782, "rew_std": 323.0878, "Agent": "A2C"}, {"env_step": 2700000, "rew": 5009.7045, "rew_std": 226.3388, "Agent": "A2C"}, {"env_step": 2730000, "rew": 4908.1905, "rew_std": 478.5281, "Agent": "A2C"}, {"env_step": 2760000, "rew": 4939.0956, "rew_std": 377.716, "Agent": "A2C"}, {"env_step": 2790000, "rew": 5165.0076, "rew_std": 268.143, "Agent": "A2C"}, {"env_step": 2820000, "rew": 5035.1886, "rew_std": 405.329, "Agent": "A2C"}, {"env_step": 2850000, "rew": 5094.7334, "rew_std": 427.7514, "Agent": "A2C"}, {"env_step": 2880000, "rew": 4926.0901, "rew_std": 387.6091, "Agent": "A2C"}, {"env_step": 2910000, "rew": 5062.6458, "rew_std": 495.6891, "Agent": "A2C"}, {"env_step": 2940000, "rew": 5111.7299, "rew_std": 400.9655, "Agent": "A2C"}, {"env_step": 2970000, "rew": 5236.7584, "rew_std": 236.6818, "Agent": "A2C"}, {"env_step": 3000000, "rew": 5064.3433, "rew_std": 284.9814, "Agent": "A2C"}] \ No newline at end of file diff --git a/examples/mujoco/benchmark/HalfCheetah-v3/result.json b/examples/mujoco/benchmark/HalfCheetah-v3/result.json new file mode 100644 index 0000000..d9a83c7 --- /dev/null +++ b/examples/mujoco/benchmark/HalfCheetah-v3/result.json @@ -0,0 +1 @@ +[{"env_step": 0, "rew": -2.027, "rew_std": 0.7352, "Agent": "SAC"}, {"env_step": 5000, "rew": 10.1839, "rew_std": 23.2072, "Agent": "SAC"}, {"env_step": 10000, "rew": 865.359, "rew_std": 486.562, "Agent": "SAC"}, {"env_step": 15000, "rew": 1814.2922, "rew_std": 570.1082, "Agent": "SAC"}, {"env_step": 20000, "rew": 2641.1068, "rew_std": 503.2532, "Agent": "SAC"}, {"env_step": 25000, "rew": 3365.0157, "rew_std": 417.5691, "Agent": "SAC"}, {"env_step": 30000, "rew": 3525.4369, "rew_std": 501.063, "Agent": "SAC"}, {"env_step": 35000, "rew": 3983.1205, "rew_std": 279.2621, "Agent": "SAC"}, {"env_step": 40000, "rew": 4326.9325, "rew_std": 213.1473, "Agent": "SAC"}, {"env_step": 45000, "rew": 4605.0875, "rew_std": 269.9916, "Agent": "SAC"}, {"env_step": 50000, "rew": 4747.2399, "rew_std": 345.0121, "Agent": "SAC"}, {"env_step": 55000, "rew": 5083.2292, "rew_std": 380.5083, "Agent": "SAC"}, {"env_step": 60000, "rew": 5316.2064, "rew_std": 330.2907, "Agent": "SAC"}, {"env_step": 65000, "rew": 5496.9468, "rew_std": 418.5496, "Agent": "SAC"}, {"env_step": 70000, "rew": 5586.9485, "rew_std": 540.2148, "Agent": "SAC"}, {"env_step": 75000, "rew": 5851.4753, "rew_std": 371.89, "Agent": "SAC"}, {"env_step": 80000, "rew": 5929.2037, "rew_std": 454.7962, "Agent": "SAC"}, {"env_step": 85000, "rew": 6106.3958, "rew_std": 438.9816, "Agent": "SAC"}, {"env_step": 90000, "rew": 6212.7888, "rew_std": 437.2369, "Agent": "SAC"}, {"env_step": 95000, "rew": 6187.6082, "rew_std": 511.4287, "Agent": "SAC"}, {"env_step": 100000, "rew": 6514.6227, "rew_std": 514.2304, "Agent": "SAC"}, {"env_step": 105000, "rew": 6485.9809, "rew_std": 597.6332, "Agent": "SAC"}, {"env_step": 110000, "rew": 6595.3192, "rew_std": 550.9856, "Agent": "SAC"}, {"env_step": 115000, "rew": 6751.1698, "rew_std": 579.1543, "Agent": "SAC"}, {"env_step": 120000, "rew": 6801.2745, "rew_std": 579.1199, "Agent": "SAC"}, {"env_step": 125000, "rew": 7076.524, "rew_std": 669.0167, "Agent": "SAC"}, {"env_step": 130000, "rew": 7088.5351, "rew_std": 673.049, "Agent": "SAC"}, {"env_step": 135000, "rew": 7229.5772, "rew_std": 688.2214, "Agent": "SAC"}, {"env_step": 140000, "rew": 7198.8854, "rew_std": 665.0043, "Agent": "SAC"}, {"env_step": 145000, "rew": 7391.4168, "rew_std": 768.3948, "Agent": "SAC"}, {"env_step": 150000, "rew": 7464.8962, "rew_std": 720.8476, "Agent": "SAC"}, {"env_step": 155000, "rew": 7437.4415, "rew_std": 763.1268, "Agent": "SAC"}, {"env_step": 160000, "rew": 7577.3921, "rew_std": 815.7664, "Agent": "SAC"}, {"env_step": 165000, "rew": 7730.9681, "rew_std": 811.9207, "Agent": "SAC"}, {"env_step": 170000, "rew": 7653.3391, "rew_std": 694.4047, "Agent": "SAC"}, {"env_step": 175000, "rew": 7943.5144, "rew_std": 887.9127, "Agent": "SAC"}, {"env_step": 180000, "rew": 7825.5208, "rew_std": 918.1944, "Agent": "SAC"}, {"env_step": 185000, "rew": 8028.0037, "rew_std": 881.683, "Agent": "SAC"}, {"env_step": 190000, "rew": 7877.3779, "rew_std": 762.8448, "Agent": "SAC"}, {"env_step": 195000, "rew": 8144.8266, "rew_std": 1004.9139, "Agent": "SAC"}, {"env_step": 200000, "rew": 8207.6336, "rew_std": 928.3063, "Agent": "SAC"}, {"env_step": 205000, "rew": 7962.9357, "rew_std": 1050.1757, "Agent": "SAC"}, {"env_step": 210000, "rew": 8309.562, "rew_std": 986.6001, "Agent": "SAC"}, {"env_step": 215000, "rew": 8434.306, "rew_std": 1014.6172, "Agent": "SAC"}, {"env_step": 220000, "rew": 8270.1339, "rew_std": 964.6572, "Agent": "SAC"}, {"env_step": 225000, "rew": 8415.4762, "rew_std": 917.3119, "Agent": "SAC"}, {"env_step": 230000, "rew": 8533.957, "rew_std": 987.645, "Agent": "SAC"}, {"env_step": 235000, "rew": 8591.5228, "rew_std": 1112.037, "Agent": "SAC"}, {"env_step": 240000, "rew": 8707.7558, "rew_std": 1031.5746, "Agent": "SAC"}, {"env_step": 245000, "rew": 8759.7748, "rew_std": 1046.6268, "Agent": "SAC"}, {"env_step": 250000, "rew": 8836.3744, "rew_std": 1070.2071, "Agent": "SAC"}, {"env_step": 255000, "rew": 8835.3932, "rew_std": 1125.7859, "Agent": "SAC"}, {"env_step": 260000, "rew": 8948.6225, "rew_std": 1104.4205, "Agent": "SAC"}, {"env_step": 265000, "rew": 8910.7138, "rew_std": 967.2416, "Agent": "SAC"}, {"env_step": 270000, "rew": 8892.1686, "rew_std": 1144.4755, "Agent": "SAC"}, {"env_step": 275000, "rew": 8966.5011, "rew_std": 1009.1316, "Agent": "SAC"}, {"env_step": 280000, "rew": 9011.7319, "rew_std": 1003.8617, "Agent": "SAC"}, {"env_step": 285000, "rew": 9067.2721, "rew_std": 1095.6016, "Agent": "SAC"}, {"env_step": 290000, "rew": 9150.2293, "rew_std": 1051.4986, "Agent": "SAC"}, {"env_step": 295000, "rew": 8819.8506, "rew_std": 1224.7711, "Agent": "SAC"}, {"env_step": 300000, "rew": 9059.2562, "rew_std": 1051.4143, "Agent": "SAC"}, {"env_step": 305000, "rew": 9241.608, "rew_std": 972.1405, "Agent": "SAC"}, {"env_step": 310000, "rew": 9317.492, "rew_std": 955.1288, "Agent": "SAC"}, {"env_step": 315000, "rew": 9108.5728, "rew_std": 933.4934, "Agent": "SAC"}, {"env_step": 320000, "rew": 9338.11, "rew_std": 807.6986, "Agent": "SAC"}, {"env_step": 325000, "rew": 9341.4086, "rew_std": 952.2938, "Agent": "SAC"}, {"env_step": 330000, "rew": 9356.795, "rew_std": 887.4584, "Agent": "SAC"}, {"env_step": 335000, "rew": 9481.7274, "rew_std": 942.1804, "Agent": "SAC"}, {"env_step": 340000, "rew": 9367.992, "rew_std": 1002.2366, "Agent": "SAC"}, {"env_step": 345000, "rew": 9424.7869, "rew_std": 793.3956, "Agent": "SAC"}, {"env_step": 350000, "rew": 9521.2021, "rew_std": 945.0578, "Agent": "SAC"}, {"env_step": 355000, "rew": 9465.8035, "rew_std": 986.819, "Agent": "SAC"}, {"env_step": 360000, "rew": 9631.8073, "rew_std": 901.8403, "Agent": "SAC"}, {"env_step": 365000, "rew": 9666.9302, "rew_std": 688.4103, "Agent": "SAC"}, {"env_step": 370000, "rew": 9845.7185, "rew_std": 776.1784, "Agent": "SAC"}, {"env_step": 375000, "rew": 9783.6146, "rew_std": 748.8536, "Agent": "SAC"}, {"env_step": 380000, "rew": 9743.993, "rew_std": 864.4312, "Agent": "SAC"}, {"env_step": 385000, "rew": 9748.2177, "rew_std": 755.897, "Agent": "SAC"}, {"env_step": 390000, "rew": 9794.1282, "rew_std": 749.9596, "Agent": "SAC"}, {"env_step": 395000, "rew": 9691.9995, "rew_std": 855.6781, "Agent": "SAC"}, {"env_step": 400000, "rew": 9908.5217, "rew_std": 901.3128, "Agent": "SAC"}, {"env_step": 405000, "rew": 10051.019, "rew_std": 774.8568, "Agent": "SAC"}, {"env_step": 410000, "rew": 10034.7663, "rew_std": 815.4708, "Agent": "SAC"}, {"env_step": 415000, "rew": 10157.1511, "rew_std": 724.3571, "Agent": "SAC"}, {"env_step": 420000, "rew": 10016.6583, "rew_std": 830.1947, "Agent": "SAC"}, {"env_step": 425000, "rew": 9990.6593, "rew_std": 783.9429, "Agent": "SAC"}, {"env_step": 430000, "rew": 9951.3972, "rew_std": 816.0467, "Agent": "SAC"}, {"env_step": 435000, "rew": 10314.0172, "rew_std": 750.6577, "Agent": "SAC"}, {"env_step": 440000, "rew": 10258.7498, "rew_std": 753.6217, "Agent": "SAC"}, {"env_step": 445000, "rew": 10039.4467, "rew_std": 1114.8295, "Agent": "SAC"}, {"env_step": 450000, "rew": 10256.0898, "rew_std": 724.7449, "Agent": "SAC"}, {"env_step": 455000, "rew": 10255.1356, "rew_std": 715.2809, "Agent": "SAC"}, {"env_step": 460000, "rew": 10313.6543, "rew_std": 670.6573, "Agent": "SAC"}, {"env_step": 465000, "rew": 10239.7386, "rew_std": 699.8731, "Agent": "SAC"}, {"env_step": 470000, "rew": 10444.677, "rew_std": 721.1596, "Agent": "SAC"}, {"env_step": 475000, "rew": 10348.6424, "rew_std": 875.3533, "Agent": "SAC"}, {"env_step": 480000, "rew": 9822.9344, "rew_std": 995.0004, "Agent": "SAC"}, {"env_step": 485000, "rew": 10373.6868, "rew_std": 676.0692, "Agent": "SAC"}, {"env_step": 490000, "rew": 10464.7314, "rew_std": 783.7438, "Agent": "SAC"}, {"env_step": 495000, "rew": 10477.8849, "rew_std": 749.5457, "Agent": "SAC"}, {"env_step": 500000, "rew": 10409.97, "rew_std": 696.7419, "Agent": "SAC"}, {"env_step": 505000, "rew": 10344.591, "rew_std": 754.885, "Agent": "SAC"}, {"env_step": 510000, "rew": 10585.7676, "rew_std": 798.5526, "Agent": "SAC"}, {"env_step": 515000, "rew": 10509.9652, "rew_std": 755.3173, "Agent": "SAC"}, {"env_step": 520000, "rew": 10578.6697, "rew_std": 767.8139, "Agent": "SAC"}, {"env_step": 525000, "rew": 10731.3679, "rew_std": 702.0967, "Agent": "SAC"}, {"env_step": 530000, "rew": 10715.2787, "rew_std": 816.2541, "Agent": "SAC"}, {"env_step": 535000, "rew": 10434.4037, "rew_std": 903.8272, "Agent": "SAC"}, {"env_step": 540000, "rew": 10402.0162, "rew_std": 865.098, "Agent": "SAC"}, {"env_step": 545000, "rew": 10469.6772, "rew_std": 1201.1587, "Agent": "SAC"}, {"env_step": 550000, "rew": 10784.2863, "rew_std": 756.5986, "Agent": "SAC"}, {"env_step": 555000, "rew": 10736.8056, "rew_std": 772.8018, "Agent": "SAC"}, {"env_step": 560000, "rew": 10693.0171, "rew_std": 808.7638, "Agent": "SAC"}, {"env_step": 565000, "rew": 10771.5095, "rew_std": 861.7875, "Agent": "SAC"}, {"env_step": 570000, "rew": 10681.0633, "rew_std": 873.4222, "Agent": "SAC"}, {"env_step": 575000, "rew": 10807.2527, "rew_std": 814.9743, "Agent": "SAC"}, {"env_step": 580000, "rew": 10937.5397, "rew_std": 819.6207, "Agent": "SAC"}, {"env_step": 585000, "rew": 10878.7778, "rew_std": 897.3742, "Agent": "SAC"}, {"env_step": 590000, "rew": 10909.0009, "rew_std": 790.7898, "Agent": "SAC"}, {"env_step": 595000, "rew": 10864.504, "rew_std": 839.9366, "Agent": "SAC"}, {"env_step": 600000, "rew": 10834.5523, "rew_std": 798.3907, "Agent": "SAC"}, {"env_step": 605000, "rew": 10965.6832, "rew_std": 782.3083, "Agent": "SAC"}, {"env_step": 610000, "rew": 10670.7502, "rew_std": 1000.5134, "Agent": "SAC"}, {"env_step": 615000, "rew": 11034.147, "rew_std": 770.7826, "Agent": "SAC"}, {"env_step": 620000, "rew": 10885.1786, "rew_std": 846.6412, "Agent": "SAC"}, {"env_step": 625000, "rew": 11171.5709, "rew_std": 842.4588, "Agent": "SAC"}, {"env_step": 630000, "rew": 11093.6177, "rew_std": 853.805, "Agent": "SAC"}, {"env_step": 635000, "rew": 11108.4159, "rew_std": 817.3521, "Agent": "SAC"}, {"env_step": 640000, "rew": 10876.0314, "rew_std": 843.1665, "Agent": "SAC"}, {"env_step": 645000, "rew": 11164.6113, "rew_std": 871.3397, "Agent": "SAC"}, {"env_step": 650000, "rew": 10951.5324, "rew_std": 844.1223, "Agent": "SAC"}, {"env_step": 655000, "rew": 10930.1218, "rew_std": 751.7118, "Agent": "SAC"}, {"env_step": 660000, "rew": 11156.2704, "rew_std": 864.4117, "Agent": "SAC"}, {"env_step": 665000, "rew": 11292.937, "rew_std": 857.9847, "Agent": "SAC"}, {"env_step": 670000, "rew": 11120.3012, "rew_std": 900.4298, "Agent": "SAC"}, {"env_step": 675000, "rew": 11238.3806, "rew_std": 799.2015, "Agent": "SAC"}, {"env_step": 680000, "rew": 11199.6556, "rew_std": 869.3494, "Agent": "SAC"}, {"env_step": 685000, "rew": 11196.8285, "rew_std": 1142.7087, "Agent": "SAC"}, {"env_step": 690000, "rew": 11376.8737, "rew_std": 843.3762, "Agent": "SAC"}, {"env_step": 695000, "rew": 11250.9709, "rew_std": 832.8242, "Agent": "SAC"}, {"env_step": 700000, "rew": 11357.1604, "rew_std": 904.5123, "Agent": "SAC"}, {"env_step": 705000, "rew": 11167.0614, "rew_std": 878.1416, "Agent": "SAC"}, {"env_step": 710000, "rew": 11142.2148, "rew_std": 849.0947, "Agent": "SAC"}, {"env_step": 715000, "rew": 11354.5586, "rew_std": 880.9652, "Agent": "SAC"}, {"env_step": 720000, "rew": 11292.1365, "rew_std": 855.3787, "Agent": "SAC"}, {"env_step": 725000, "rew": 11397.1391, "rew_std": 849.8348, "Agent": "SAC"}, {"env_step": 730000, "rew": 11363.1654, "rew_std": 865.1339, "Agent": "SAC"}, {"env_step": 735000, "rew": 11401.6934, "rew_std": 985.0648, "Agent": "SAC"}, {"env_step": 740000, "rew": 11457.9921, "rew_std": 882.5772, "Agent": "SAC"}, {"env_step": 745000, "rew": 11263.8964, "rew_std": 861.9337, "Agent": "SAC"}, {"env_step": 750000, "rew": 11329.132, "rew_std": 870.3124, "Agent": "SAC"}, {"env_step": 755000, "rew": 11432.5841, "rew_std": 878.3916, "Agent": "SAC"}, {"env_step": 760000, "rew": 11366.4875, "rew_std": 1066.7255, "Agent": "SAC"}, {"env_step": 765000, "rew": 11235.2494, "rew_std": 1250.3021, "Agent": "SAC"}, {"env_step": 770000, "rew": 11393.2447, "rew_std": 956.0431, "Agent": "SAC"}, {"env_step": 775000, "rew": 11499.3649, "rew_std": 903.8219, "Agent": "SAC"}, {"env_step": 780000, "rew": 11377.0247, "rew_std": 857.1325, "Agent": "SAC"}, {"env_step": 785000, "rew": 11420.9959, "rew_std": 945.2315, "Agent": "SAC"}, {"env_step": 790000, "rew": 11381.5919, "rew_std": 870.4013, "Agent": "SAC"}, {"env_step": 795000, "rew": 11440.576, "rew_std": 898.4534, "Agent": "SAC"}, {"env_step": 800000, "rew": 11676.8494, "rew_std": 971.5681, "Agent": "SAC"}, {"env_step": 805000, "rew": 11607.9284, "rew_std": 904.6303, "Agent": "SAC"}, {"env_step": 810000, "rew": 11336.3882, "rew_std": 967.6672, "Agent": "SAC"}, {"env_step": 815000, "rew": 11037.0411, "rew_std": 1891.8502, "Agent": "SAC"}, {"env_step": 820000, "rew": 11702.6572, "rew_std": 930.5494, "Agent": "SAC"}, {"env_step": 825000, "rew": 11571.8939, "rew_std": 964.7258, "Agent": "SAC"}, {"env_step": 830000, "rew": 11714.3621, "rew_std": 953.5695, "Agent": "SAC"}, {"env_step": 835000, "rew": 11484.7875, "rew_std": 919.6083, "Agent": "SAC"}, {"env_step": 840000, "rew": 11653.8103, "rew_std": 892.0375, "Agent": "SAC"}, {"env_step": 845000, "rew": 11684.4042, "rew_std": 866.6908, "Agent": "SAC"}, {"env_step": 850000, "rew": 11681.3375, "rew_std": 961.4558, "Agent": "SAC"}, {"env_step": 855000, "rew": 11587.8436, "rew_std": 933.4399, "Agent": "SAC"}, {"env_step": 860000, "rew": 11661.2521, "rew_std": 1040.2257, "Agent": "SAC"}, {"env_step": 865000, "rew": 11488.6788, "rew_std": 1063.185, "Agent": "SAC"}, {"env_step": 870000, "rew": 11540.2953, "rew_std": 1037.9443, "Agent": "SAC"}, {"env_step": 875000, "rew": 11841.9591, "rew_std": 963.0978, "Agent": "SAC"}, {"env_step": 880000, "rew": 11793.1006, "rew_std": 896.7818, "Agent": "SAC"}, {"env_step": 885000, "rew": 11874.787, "rew_std": 980.7893, "Agent": "SAC"}, {"env_step": 890000, "rew": 11648.9695, "rew_std": 905.3032, "Agent": "SAC"}, {"env_step": 895000, "rew": 11912.3981, "rew_std": 944.7161, "Agent": "SAC"}, {"env_step": 900000, "rew": 12012.0246, "rew_std": 1032.0492, "Agent": "SAC"}, {"env_step": 905000, "rew": 11748.9588, "rew_std": 961.4382, "Agent": "SAC"}, {"env_step": 910000, "rew": 11786.2655, "rew_std": 1031.0671, "Agent": "SAC"}, {"env_step": 915000, "rew": 11219.5542, "rew_std": 1884.3568, "Agent": "SAC"}, {"env_step": 920000, "rew": 11846.4572, "rew_std": 1007.202, "Agent": "SAC"}, {"env_step": 925000, "rew": 11506.1745, "rew_std": 1096.2194, "Agent": "SAC"}, {"env_step": 930000, "rew": 11861.6563, "rew_std": 1074.5994, "Agent": "SAC"}, {"env_step": 935000, "rew": 11753.2089, "rew_std": 1013.6127, "Agent": "SAC"}, {"env_step": 940000, "rew": 11921.9789, "rew_std": 939.7152, "Agent": "SAC"}, {"env_step": 945000, "rew": 11950.4468, "rew_std": 949.9662, "Agent": "SAC"}, {"env_step": 950000, "rew": 12028.8625, "rew_std": 1032.3731, "Agent": "SAC"}, {"env_step": 955000, "rew": 11937.5016, "rew_std": 957.3345, "Agent": "SAC"}, {"env_step": 960000, "rew": 11984.8677, "rew_std": 1016.5514, "Agent": "SAC"}, {"env_step": 965000, "rew": 11962.7994, "rew_std": 997.6641, "Agent": "SAC"}, {"env_step": 970000, "rew": 12027.9757, "rew_std": 941.3748, "Agent": "SAC"}, {"env_step": 975000, "rew": 11939.0175, "rew_std": 999.1897, "Agent": "SAC"}, {"env_step": 980000, "rew": 11913.8985, "rew_std": 916.6498, "Agent": "SAC"}, {"env_step": 985000, "rew": 11802.1434, "rew_std": 1129.3786, "Agent": "SAC"}, {"env_step": 990000, "rew": 12138.7987, "rew_std": 1049.3246, "Agent": "SAC"}, {"env_step": 995000, "rew": 12130.1697, "rew_std": 993.185, "Agent": "SAC"}, {"env_step": 1000000, "rew": 12121.8566, "rew_std": 1006.2108, "Agent": "SAC"}, {"env_step": 1005000, "rew": 11942.5856, "rew_std": 991.2353, "Agent": "SAC"}, {"env_step": 1010000, "rew": 12005.1579, "rew_std": 1045.1113, "Agent": "SAC"}, {"env_step": 1015000, "rew": 12041.7859, "rew_std": 998.4825, "Agent": "SAC"}, {"env_step": 1020000, "rew": 12013.6874, "rew_std": 1026.4441, "Agent": "SAC"}, {"env_step": 1025000, "rew": 12065.7275, "rew_std": 1012.3839, "Agent": "SAC"}, {"env_step": 1030000, "rew": 11714.3521, "rew_std": 1212.8575, "Agent": "SAC"}, {"env_step": 1035000, "rew": 12218.3614, "rew_std": 1082.4594, "Agent": "SAC"}, {"env_step": 1040000, "rew": 12184.5401, "rew_std": 1020.1823, "Agent": "SAC"}, {"env_step": 1045000, "rew": 12244.2109, "rew_std": 1030.6247, "Agent": "SAC"}, {"env_step": 1050000, "rew": 12202.9869, "rew_std": 1078.7199, "Agent": "SAC"}, {"env_step": 1055000, "rew": 12239.3944, "rew_std": 1019.3269, "Agent": "SAC"}, {"env_step": 1060000, "rew": 11997.9335, "rew_std": 914.4304, "Agent": "SAC"}, {"env_step": 1065000, "rew": 12149.6644, "rew_std": 893.8064, "Agent": "SAC"}, {"env_step": 1070000, "rew": 12175.7625, "rew_std": 1044.152, "Agent": "SAC"}, {"env_step": 1075000, "rew": 12346.2323, "rew_std": 1005.64, "Agent": "SAC"}, {"env_step": 1080000, "rew": 12362.0626, "rew_std": 1034.2557, "Agent": "SAC"}, {"env_step": 1085000, "rew": 12180.2346, "rew_std": 1023.6408, "Agent": "SAC"}, {"env_step": 1090000, "rew": 12104.2195, "rew_std": 1141.1273, "Agent": "SAC"}, {"env_step": 1095000, "rew": 12191.2541, "rew_std": 1113.5694, "Agent": "SAC"}, {"env_step": 1100000, "rew": 12395.9566, "rew_std": 952.733, "Agent": "SAC"}, {"env_step": 1105000, "rew": 12347.7701, "rew_std": 801.054, "Agent": "SAC"}, {"env_step": 1110000, "rew": 12235.5543, "rew_std": 1035.2947, "Agent": "SAC"}, {"env_step": 1115000, "rew": 12449.6886, "rew_std": 994.0012, "Agent": "SAC"}, {"env_step": 1120000, "rew": 12209.3594, "rew_std": 855.504, "Agent": "SAC"}, {"env_step": 1125000, "rew": 12242.4778, "rew_std": 958.1728, "Agent": "SAC"}, {"env_step": 1130000, "rew": 12435.6339, "rew_std": 713.4702, "Agent": "SAC"}, {"env_step": 1135000, "rew": 12379.179, "rew_std": 955.365, "Agent": "SAC"}, {"env_step": 1140000, "rew": 12235.5068, "rew_std": 1268.6864, "Agent": "SAC"}, {"env_step": 1145000, "rew": 12097.1035, "rew_std": 873.944, "Agent": "SAC"}, {"env_step": 1150000, "rew": 12531.4431, "rew_std": 932.6429, "Agent": "SAC"}, {"env_step": 1155000, "rew": 12377.1813, "rew_std": 815.1965, "Agent": "SAC"}, {"env_step": 1160000, "rew": 12598.7106, "rew_std": 843.0569, "Agent": "SAC"}, {"env_step": 1165000, "rew": 12420.8783, "rew_std": 869.0023, "Agent": "SAC"}, {"env_step": 1170000, "rew": 12546.8189, "rew_std": 872.5977, "Agent": "SAC"}, {"env_step": 1175000, "rew": 12338.7278, "rew_std": 934.1012, "Agent": "SAC"}, {"env_step": 1180000, "rew": 12441.9147, "rew_std": 970.1274, "Agent": "SAC"}, {"env_step": 1185000, "rew": 12395.7239, "rew_std": 856.8003, "Agent": "SAC"}, {"env_step": 1190000, "rew": 12693.0426, "rew_std": 841.1914, "Agent": "SAC"}, {"env_step": 1195000, "rew": 12438.0021, "rew_std": 1017.4289, "Agent": "SAC"}, {"env_step": 1200000, "rew": 12248.8422, "rew_std": 860.0051, "Agent": "SAC"}, {"env_step": 1205000, "rew": 12643.7048, "rew_std": 952.921, "Agent": "SAC"}, {"env_step": 1210000, "rew": 12449.039, "rew_std": 1083.0253, "Agent": "SAC"}, {"env_step": 1215000, "rew": 12576.7585, "rew_std": 913.4922, "Agent": "SAC"}, {"env_step": 1220000, "rew": 12584.8301, "rew_std": 883.0901, "Agent": "SAC"}, {"env_step": 1225000, "rew": 12716.9272, "rew_std": 996.5454, "Agent": "SAC"}, {"env_step": 1230000, "rew": 12436.8757, "rew_std": 882.0322, "Agent": "SAC"}, {"env_step": 1235000, "rew": 12590.1792, "rew_std": 886.9334, "Agent": "SAC"}, {"env_step": 1240000, "rew": 12652.7922, "rew_std": 932.0824, "Agent": "SAC"}, {"env_step": 1245000, "rew": 12458.9671, "rew_std": 1062.3939, "Agent": "SAC"}, {"env_step": 1250000, "rew": 12533.7516, "rew_std": 859.9165, "Agent": "SAC"}, {"env_step": 0, "rew": -2.027, "rew_std": 0.7352, "Agent": "DDPG"}, {"env_step": 5000, "rew": -28.8271, "rew_std": 334.3419, "Agent": "DDPG"}, {"env_step": 10000, "rew": 1412.5232, "rew_std": 641.5603, "Agent": "DDPG"}, {"env_step": 15000, "rew": 1981.823, "rew_std": 561.1781, "Agent": "DDPG"}, {"env_step": 20000, "rew": 2379.8574, "rew_std": 785.2131, "Agent": "DDPG"}, {"env_step": 25000, "rew": 3125.7271, "rew_std": 479.5881, "Agent": "DDPG"}, {"env_step": 30000, "rew": 3456.9745, "rew_std": 433.4561, "Agent": "DDPG"}, {"env_step": 35000, "rew": 3823.9994, "rew_std": 779.0331, "Agent": "DDPG"}, {"env_step": 40000, "rew": 3978.0427, "rew_std": 719.3501, "Agent": "DDPG"}, {"env_step": 45000, "rew": 4546.8822, "rew_std": 620.1597, "Agent": "DDPG"}, {"env_step": 50000, "rew": 4523.1437, "rew_std": 791.7028, "Agent": "DDPG"}, {"env_step": 55000, "rew": 4710.7945, "rew_std": 912.7164, "Agent": "DDPG"}, {"env_step": 60000, "rew": 4860.8532, "rew_std": 981.0827, "Agent": "DDPG"}, {"env_step": 65000, "rew": 4937.9971, "rew_std": 1051.3911, "Agent": "DDPG"}, {"env_step": 70000, "rew": 4991.6168, "rew_std": 883.5144, "Agent": "DDPG"}, {"env_step": 75000, "rew": 5598.5992, "rew_std": 676.2098, "Agent": "DDPG"}, {"env_step": 80000, "rew": 5188.1598, "rew_std": 1077.1823, "Agent": "DDPG"}, {"env_step": 85000, "rew": 5752.0384, "rew_std": 932.4943, "Agent": "DDPG"}, {"env_step": 90000, "rew": 5805.2834, "rew_std": 1228.4622, "Agent": "DDPG"}, {"env_step": 95000, "rew": 6139.6541, "rew_std": 812.3183, "Agent": "DDPG"}, {"env_step": 100000, "rew": 6308.281, "rew_std": 1011.4719, "Agent": "DDPG"}, {"env_step": 105000, "rew": 6578.4465, "rew_std": 902.1652, "Agent": "DDPG"}, {"env_step": 110000, "rew": 6225.5968, "rew_std": 628.945, "Agent": "DDPG"}, {"env_step": 115000, "rew": 6618.0579, "rew_std": 792.9479, "Agent": "DDPG"}, {"env_step": 120000, "rew": 6636.5746, "rew_std": 758.4807, "Agent": "DDPG"}, {"env_step": 125000, "rew": 6726.8723, "rew_std": 746.8756, "Agent": "DDPG"}, {"env_step": 130000, "rew": 6426.0841, "rew_std": 1312.1471, "Agent": "DDPG"}, {"env_step": 135000, "rew": 5961.9648, "rew_std": 2287.1167, "Agent": "DDPG"}, {"env_step": 140000, "rew": 6294.7688, "rew_std": 1638.965, "Agent": "DDPG"}, {"env_step": 145000, "rew": 6992.672, "rew_std": 834.1884, "Agent": "DDPG"}, {"env_step": 150000, "rew": 6974.0589, "rew_std": 802.4812, "Agent": "DDPG"}, {"env_step": 155000, "rew": 7314.7522, "rew_std": 848.3754, "Agent": "DDPG"}, {"env_step": 160000, "rew": 7296.0335, "rew_std": 999.8537, "Agent": "DDPG"}, {"env_step": 165000, "rew": 7394.7493, "rew_std": 758.0624, "Agent": "DDPG"}, {"env_step": 170000, "rew": 7444.2435, "rew_std": 896.1343, "Agent": "DDPG"}, {"env_step": 175000, "rew": 7506.2172, "rew_std": 939.2738, "Agent": "DDPG"}, {"env_step": 180000, "rew": 7792.2261, "rew_std": 922.8894, "Agent": "DDPG"}, {"env_step": 185000, "rew": 7772.0288, "rew_std": 780.9202, "Agent": "DDPG"}, {"env_step": 190000, "rew": 7947.7237, "rew_std": 855.0877, "Agent": "DDPG"}, {"env_step": 195000, "rew": 7787.8832, "rew_std": 799.944, "Agent": "DDPG"}, {"env_step": 200000, "rew": 7913.7071, "rew_std": 1074.6718, "Agent": "DDPG"}, {"env_step": 205000, "rew": 7950.1875, "rew_std": 1026.8272, "Agent": "DDPG"}, {"env_step": 210000, "rew": 7572.8741, "rew_std": 1350.0385, "Agent": "DDPG"}, {"env_step": 215000, "rew": 7531.8723, "rew_std": 2504.3457, "Agent": "DDPG"}, {"env_step": 220000, "rew": 8093.4657, "rew_std": 1265.1079, "Agent": "DDPG"}, {"env_step": 225000, "rew": 8232.0487, "rew_std": 938.8681, "Agent": "DDPG"}, {"env_step": 230000, "rew": 8453.1188, "rew_std": 672.4392, "Agent": "DDPG"}, {"env_step": 235000, "rew": 8562.3664, "rew_std": 1045.5171, "Agent": "DDPG"}, {"env_step": 240000, "rew": 8503.9181, "rew_std": 803.2379, "Agent": "DDPG"}, {"env_step": 245000, "rew": 7906.8492, "rew_std": 2653.9952, "Agent": "DDPG"}, {"env_step": 250000, "rew": 7774.7352, "rew_std": 2889.966, "Agent": "DDPG"}, {"env_step": 255000, "rew": 7570.7543, "rew_std": 2772.8448, "Agent": "DDPG"}, {"env_step": 260000, "rew": 8732.4448, "rew_std": 760.7779, "Agent": "DDPG"}, {"env_step": 265000, "rew": 8753.7309, "rew_std": 656.8576, "Agent": "DDPG"}, {"env_step": 270000, "rew": 8893.9039, "rew_std": 633.8578, "Agent": "DDPG"}, {"env_step": 275000, "rew": 9089.5602, "rew_std": 764.8756, "Agent": "DDPG"}, {"env_step": 280000, "rew": 8964.6522, "rew_std": 642.0339, "Agent": "DDPG"}, {"env_step": 285000, "rew": 8901.8237, "rew_std": 528.058, "Agent": "DDPG"}, {"env_step": 290000, "rew": 8804.5557, "rew_std": 626.7264, "Agent": "DDPG"}, {"env_step": 295000, "rew": 9015.7802, "rew_std": 676.9585, "Agent": "DDPG"}, {"env_step": 300000, "rew": 9137.0428, "rew_std": 784.2173, "Agent": "DDPG"}, {"env_step": 305000, "rew": 9330.9567, "rew_std": 671.3886, "Agent": "DDPG"}, {"env_step": 310000, "rew": 9409.5571, "rew_std": 721.7062, "Agent": "DDPG"}, {"env_step": 315000, "rew": 9341.3278, "rew_std": 651.8969, "Agent": "DDPG"}, {"env_step": 320000, "rew": 9425.0942, "rew_std": 645.917, "Agent": "DDPG"}, {"env_step": 325000, "rew": 9431.0494, "rew_std": 869.0498, "Agent": "DDPG"}, {"env_step": 330000, "rew": 9086.6945, "rew_std": 688.3953, "Agent": "DDPG"}, {"env_step": 335000, "rew": 9417.2718, "rew_std": 746.1302, "Agent": "DDPG"}, {"env_step": 340000, "rew": 9602.646, "rew_std": 701.0568, "Agent": "DDPG"}, {"env_step": 345000, "rew": 9777.0993, "rew_std": 747.7249, "Agent": "DDPG"}, {"env_step": 350000, "rew": 9554.0485, "rew_std": 722.7574, "Agent": "DDPG"}, {"env_step": 355000, "rew": 9594.6082, "rew_std": 755.1396, "Agent": "DDPG"}, {"env_step": 360000, "rew": 9544.3629, "rew_std": 688.7421, "Agent": "DDPG"}, {"env_step": 365000, "rew": 9405.8409, "rew_std": 582.176, "Agent": "DDPG"}, {"env_step": 370000, "rew": 9343.8184, "rew_std": 670.104, "Agent": "DDPG"}, {"env_step": 375000, "rew": 9658.2741, "rew_std": 533.29, "Agent": "DDPG"}, {"env_step": 380000, "rew": 9728.9095, "rew_std": 538.4292, "Agent": "DDPG"}, {"env_step": 385000, "rew": 9820.3723, "rew_std": 629.1376, "Agent": "DDPG"}, {"env_step": 390000, "rew": 9865.2476, "rew_std": 547.6719, "Agent": "DDPG"}, {"env_step": 395000, "rew": 10002.046, "rew_std": 736.7768, "Agent": "DDPG"}, {"env_step": 400000, "rew": 9945.8354, "rew_std": 547.7547, "Agent": "DDPG"}, {"env_step": 405000, "rew": 9959.5984, "rew_std": 546.7624, "Agent": "DDPG"}, {"env_step": 410000, "rew": 10083.6288, "rew_std": 541.2135, "Agent": "DDPG"}, {"env_step": 415000, "rew": 9978.6106, "rew_std": 680.8054, "Agent": "DDPG"}, {"env_step": 420000, "rew": 10000.9096, "rew_std": 786.5215, "Agent": "DDPG"}, {"env_step": 425000, "rew": 10019.7757, "rew_std": 435.3879, "Agent": "DDPG"}, {"env_step": 430000, "rew": 9872.1755, "rew_std": 779.0172, "Agent": "DDPG"}, {"env_step": 435000, "rew": 10025.4855, "rew_std": 482.0434, "Agent": "DDPG"}, {"env_step": 440000, "rew": 10240.6122, "rew_std": 656.8871, "Agent": "DDPG"}, {"env_step": 445000, "rew": 10257.9228, "rew_std": 762.7639, "Agent": "DDPG"}, {"env_step": 450000, "rew": 10107.0443, "rew_std": 554.7625, "Agent": "DDPG"}, {"env_step": 455000, "rew": 10270.8498, "rew_std": 524.4578, "Agent": "DDPG"}, {"env_step": 460000, "rew": 10353.1613, "rew_std": 583.0093, "Agent": "DDPG"}, {"env_step": 465000, "rew": 10209.6424, "rew_std": 594.339, "Agent": "DDPG"}, {"env_step": 470000, "rew": 10087.5832, "rew_std": 663.6491, "Agent": "DDPG"}, {"env_step": 475000, "rew": 10517.1995, "rew_std": 632.7956, "Agent": "DDPG"}, {"env_step": 480000, "rew": 10349.4821, "rew_std": 536.1071, "Agent": "DDPG"}, {"env_step": 485000, "rew": 10127.5573, "rew_std": 691.1824, "Agent": "DDPG"}, {"env_step": 490000, "rew": 10253.503, "rew_std": 848.9819, "Agent": "DDPG"}, {"env_step": 495000, "rew": 10441.2324, "rew_std": 711.643, "Agent": "DDPG"}, {"env_step": 500000, "rew": 10440.0106, "rew_std": 593.0907, "Agent": "DDPG"}, {"env_step": 505000, "rew": 10530.2291, "rew_std": 587.6, "Agent": "DDPG"}, {"env_step": 510000, "rew": 10117.4034, "rew_std": 800.7787, "Agent": "DDPG"}, {"env_step": 515000, "rew": 10467.5302, "rew_std": 791.5499, "Agent": "DDPG"}, {"env_step": 520000, "rew": 10716.2178, "rew_std": 604.1685, "Agent": "DDPG"}, {"env_step": 525000, "rew": 10378.1373, "rew_std": 634.8955, "Agent": "DDPG"}, {"env_step": 530000, "rew": 10637.9629, "rew_std": 679.0302, "Agent": "DDPG"}, {"env_step": 535000, "rew": 10373.6315, "rew_std": 510.653, "Agent": "DDPG"}, {"env_step": 540000, "rew": 10604.1965, "rew_std": 628.5797, "Agent": "DDPG"}, {"env_step": 545000, "rew": 10653.719, "rew_std": 416.4147, "Agent": "DDPG"}, {"env_step": 550000, "rew": 10511.583, "rew_std": 587.2064, "Agent": "DDPG"}, {"env_step": 555000, "rew": 10560.0644, "rew_std": 646.5575, "Agent": "DDPG"}, {"env_step": 560000, "rew": 10754.83, "rew_std": 588.194, "Agent": "DDPG"}, {"env_step": 565000, "rew": 10668.4106, "rew_std": 539.8896, "Agent": "DDPG"}, {"env_step": 570000, "rew": 10749.6936, "rew_std": 447.1471, "Agent": "DDPG"}, {"env_step": 575000, "rew": 10918.2499, "rew_std": 640.5309, "Agent": "DDPG"}, {"env_step": 580000, "rew": 10610.7114, "rew_std": 643.848, "Agent": "DDPG"}, {"env_step": 585000, "rew": 10678.5014, "rew_std": 419.7959, "Agent": "DDPG"}, {"env_step": 590000, "rew": 10629.0255, "rew_std": 566.6714, "Agent": "DDPG"}, {"env_step": 595000, "rew": 10886.1181, "rew_std": 636.4089, "Agent": "DDPG"}, {"env_step": 600000, "rew": 10742.807, "rew_std": 927.385, "Agent": "DDPG"}, {"env_step": 605000, "rew": 10945.2297, "rew_std": 613.0327, "Agent": "DDPG"}, {"env_step": 610000, "rew": 10709.0516, "rew_std": 721.4677, "Agent": "DDPG"}, {"env_step": 615000, "rew": 11037.4454, "rew_std": 506.5949, "Agent": "DDPG"}, {"env_step": 620000, "rew": 11016.5806, "rew_std": 569.935, "Agent": "DDPG"}, {"env_step": 625000, "rew": 10885.3285, "rew_std": 659.2231, "Agent": "DDPG"}, {"env_step": 630000, "rew": 10900.9946, "rew_std": 707.7155, "Agent": "DDPG"}, {"env_step": 635000, "rew": 10942.9195, "rew_std": 636.7993, "Agent": "DDPG"}, {"env_step": 640000, "rew": 10882.5497, "rew_std": 841.9896, "Agent": "DDPG"}, {"env_step": 645000, "rew": 11028.7324, "rew_std": 506.2372, "Agent": "DDPG"}, {"env_step": 650000, "rew": 11011.8379, "rew_std": 618.0482, "Agent": "DDPG"}, {"env_step": 655000, "rew": 11139.0448, "rew_std": 467.1458, "Agent": "DDPG"}, {"env_step": 660000, "rew": 10950.6443, "rew_std": 616.6183, "Agent": "DDPG"}, {"env_step": 665000, "rew": 11023.1547, "rew_std": 665.6591, "Agent": "DDPG"}, {"env_step": 670000, "rew": 11194.2498, "rew_std": 574.0943, "Agent": "DDPG"}, {"env_step": 675000, "rew": 11059.1481, "rew_std": 584.0984, "Agent": "DDPG"}, {"env_step": 680000, "rew": 11154.7111, "rew_std": 658.793, "Agent": "DDPG"}, {"env_step": 685000, "rew": 11019.586, "rew_std": 645.6744, "Agent": "DDPG"}, {"env_step": 690000, "rew": 11172.7984, "rew_std": 594.8887, "Agent": "DDPG"}, {"env_step": 695000, "rew": 11071.6482, "rew_std": 451.4383, "Agent": "DDPG"}, {"env_step": 700000, "rew": 11236.911, "rew_std": 467.8947, "Agent": "DDPG"}, {"env_step": 705000, "rew": 11105.3196, "rew_std": 567.6877, "Agent": "DDPG"}, {"env_step": 710000, "rew": 11044.5712, "rew_std": 582.38, "Agent": "DDPG"}, {"env_step": 715000, "rew": 11403.593, "rew_std": 594.2401, "Agent": "DDPG"}, {"env_step": 720000, "rew": 11207.3252, "rew_std": 552.8052, "Agent": "DDPG"}, {"env_step": 725000, "rew": 11142.915, "rew_std": 569.0722, "Agent": "DDPG"}, {"env_step": 730000, "rew": 11156.9241, "rew_std": 510.7954, "Agent": "DDPG"}, {"env_step": 735000, "rew": 11116.1692, "rew_std": 432.8838, "Agent": "DDPG"}, {"env_step": 740000, "rew": 11067.6984, "rew_std": 767.6581, "Agent": "DDPG"}, {"env_step": 745000, "rew": 11300.9546, "rew_std": 607.1508, "Agent": "DDPG"}, {"env_step": 750000, "rew": 11083.3306, "rew_std": 603.7046, "Agent": "DDPG"}, {"env_step": 755000, "rew": 11192.8106, "rew_std": 493.0479, "Agent": "DDPG"}, {"env_step": 760000, "rew": 11286.6854, "rew_std": 522.0469, "Agent": "DDPG"}, {"env_step": 765000, "rew": 11233.9554, "rew_std": 463.6113, "Agent": "DDPG"}, {"env_step": 770000, "rew": 11439.1584, "rew_std": 585.3903, "Agent": "DDPG"}, {"env_step": 775000, "rew": 11361.0733, "rew_std": 546.9008, "Agent": "DDPG"}, {"env_step": 780000, "rew": 11468.0895, "rew_std": 552.944, "Agent": "DDPG"}, {"env_step": 785000, "rew": 11191.989, "rew_std": 763.2787, "Agent": "DDPG"}, {"env_step": 790000, "rew": 11370.4669, "rew_std": 596.5506, "Agent": "DDPG"}, {"env_step": 795000, "rew": 11218.5534, "rew_std": 525.4173, "Agent": "DDPG"}, {"env_step": 800000, "rew": 11289.6965, "rew_std": 517.6745, "Agent": "DDPG"}, {"env_step": 805000, "rew": 11147.309, "rew_std": 621.4135, "Agent": "DDPG"}, {"env_step": 810000, "rew": 11321.261, "rew_std": 395.6105, "Agent": "DDPG"}, {"env_step": 815000, "rew": 11385.979, "rew_std": 557.1665, "Agent": "DDPG"}, {"env_step": 820000, "rew": 11481.979, "rew_std": 490.3213, "Agent": "DDPG"}, {"env_step": 825000, "rew": 11310.0771, "rew_std": 391.0254, "Agent": "DDPG"}, {"env_step": 830000, "rew": 11467.7414, "rew_std": 609.875, "Agent": "DDPG"}, {"env_step": 835000, "rew": 11366.6416, "rew_std": 575.2764, "Agent": "DDPG"}, {"env_step": 840000, "rew": 11284.9116, "rew_std": 708.7849, "Agent": "DDPG"}, {"env_step": 845000, "rew": 11319.6365, "rew_std": 689.111, "Agent": "DDPG"}, {"env_step": 850000, "rew": 11282.7886, "rew_std": 396.2033, "Agent": "DDPG"}, {"env_step": 855000, "rew": 11381.5945, "rew_std": 471.7505, "Agent": "DDPG"}, {"env_step": 860000, "rew": 11411.786, "rew_std": 547.2996, "Agent": "DDPG"}, {"env_step": 865000, "rew": 11454.9553, "rew_std": 563.5031, "Agent": "DDPG"}, {"env_step": 870000, "rew": 11457.6559, "rew_std": 556.0714, "Agent": "DDPG"}, {"env_step": 875000, "rew": 11472.6449, "rew_std": 638.8048, "Agent": "DDPG"}, {"env_step": 880000, "rew": 11465.5412, "rew_std": 705.4152, "Agent": "DDPG"}, {"env_step": 885000, "rew": 11549.7596, "rew_std": 514.3814, "Agent": "DDPG"}, {"env_step": 890000, "rew": 11235.0108, "rew_std": 834.1102, "Agent": "DDPG"}, {"env_step": 895000, "rew": 11622.4758, "rew_std": 536.3827, "Agent": "DDPG"}, {"env_step": 900000, "rew": 11668.2049, "rew_std": 425.4828, "Agent": "DDPG"}, {"env_step": 905000, "rew": 11616.5358, "rew_std": 520.914, "Agent": "DDPG"}, {"env_step": 910000, "rew": 11570.1639, "rew_std": 521.4163, "Agent": "DDPG"}, {"env_step": 915000, "rew": 11541.2839, "rew_std": 554.1431, "Agent": "DDPG"}, {"env_step": 920000, "rew": 11296.8198, "rew_std": 749.0817, "Agent": "DDPG"}, {"env_step": 925000, "rew": 11646.0875, "rew_std": 521.0823, "Agent": "DDPG"}, {"env_step": 930000, "rew": 11592.4523, "rew_std": 562.7359, "Agent": "DDPG"}, {"env_step": 935000, "rew": 11427.8314, "rew_std": 530.3684, "Agent": "DDPG"}, {"env_step": 940000, "rew": 11597.167, "rew_std": 618.9079, "Agent": "DDPG"}, {"env_step": 945000, "rew": 11580.1866, "rew_std": 415.1078, "Agent": "DDPG"}, {"env_step": 950000, "rew": 11642.5185, "rew_std": 518.7493, "Agent": "DDPG"}, {"env_step": 955000, "rew": 11612.4348, "rew_std": 567.302, "Agent": "DDPG"}, {"env_step": 960000, "rew": 11654.0423, "rew_std": 636.8007, "Agent": "DDPG"}, {"env_step": 965000, "rew": 11365.9275, "rew_std": 571.5053, "Agent": "DDPG"}, {"env_step": 970000, "rew": 11294.5028, "rew_std": 518.7431, "Agent": "DDPG"}, {"env_step": 975000, "rew": 11529.7316, "rew_std": 460.8124, "Agent": "DDPG"}, {"env_step": 980000, "rew": 11718.7478, "rew_std": 465.5913, "Agent": "DDPG"}, {"env_step": 985000, "rew": 11458.796, "rew_std": 711.0699, "Agent": "DDPG"}, {"env_step": 990000, "rew": 11604.3687, "rew_std": 652.2274, "Agent": "DDPG"}, {"env_step": 995000, "rew": 11693.6323, "rew_std": 500.5428, "Agent": "DDPG"}, {"env_step": 1000000, "rew": 11652.8908, "rew_std": 430.4893, "Agent": "DDPG"}, {"env_step": 1005000, "rew": 11678.9465, "rew_std": 431.8784, "Agent": "DDPG"}, {"env_step": 1010000, "rew": 11774.2687, "rew_std": 529.5313, "Agent": "DDPG"}, {"env_step": 1015000, "rew": 11708.9961, "rew_std": 340.2499, "Agent": "DDPG"}, {"env_step": 1020000, "rew": 11709.5112, "rew_std": 593.4881, "Agent": "DDPG"}, {"env_step": 1025000, "rew": 11478.2603, "rew_std": 583.3108, "Agent": "DDPG"}, {"env_step": 1030000, "rew": 11735.4503, "rew_std": 622.3741, "Agent": "DDPG"}, {"env_step": 1035000, "rew": 11926.9122, "rew_std": 441.7554, "Agent": "DDPG"}, {"env_step": 1040000, "rew": 11727.8934, "rew_std": 626.3022, "Agent": "DDPG"}, {"env_step": 1045000, "rew": 11860.2139, "rew_std": 306.5245, "Agent": "DDPG"}, {"env_step": 1050000, "rew": 11766.3819, "rew_std": 402.2232, "Agent": "DDPG"}, {"env_step": 1055000, "rew": 11788.8851, "rew_std": 607.4994, "Agent": "DDPG"}, {"env_step": 1060000, "rew": 12029.9967, "rew_std": 457.2206, "Agent": "DDPG"}, {"env_step": 1065000, "rew": 11764.5045, "rew_std": 547.902, "Agent": "DDPG"}, {"env_step": 1070000, "rew": 11685.009, "rew_std": 708.9527, "Agent": "DDPG"}, {"env_step": 1075000, "rew": 11612.3431, "rew_std": 411.729, "Agent": "DDPG"}, {"env_step": 1080000, "rew": 11726.5448, "rew_std": 479.7486, "Agent": "DDPG"}, {"env_step": 1085000, "rew": 11942.8728, "rew_std": 481.9975, "Agent": "DDPG"}, {"env_step": 1090000, "rew": 11991.8823, "rew_std": 452.0352, "Agent": "DDPG"}, {"env_step": 1095000, "rew": 11780.3107, "rew_std": 337.2172, "Agent": "DDPG"}, {"env_step": 1100000, "rew": 11866.922, "rew_std": 550.2163, "Agent": "DDPG"}, {"env_step": 1105000, "rew": 11797.3379, "rew_std": 290.9405, "Agent": "DDPG"}, {"env_step": 1110000, "rew": 11751.0919, "rew_std": 432.3337, "Agent": "DDPG"}, {"env_step": 1115000, "rew": 11676.7626, "rew_std": 573.8274, "Agent": "DDPG"}, {"env_step": 1120000, "rew": 11926.8204, "rew_std": 462.6329, "Agent": "DDPG"}, {"env_step": 1125000, "rew": 11934.5671, "rew_std": 449.9755, "Agent": "DDPG"}, {"env_step": 1130000, "rew": 11872.5645, "rew_std": 571.6153, "Agent": "DDPG"}, {"env_step": 1135000, "rew": 11845.6371, "rew_std": 434.0573, "Agent": "DDPG"}, {"env_step": 1140000, "rew": 11960.4539, "rew_std": 338.6846, "Agent": "DDPG"}, {"env_step": 1145000, "rew": 12070.2261, "rew_std": 428.1063, "Agent": "DDPG"}, {"env_step": 1150000, "rew": 11943.9107, "rew_std": 461.9049, "Agent": "DDPG"}, {"env_step": 1155000, "rew": 11802.2626, "rew_std": 481.006, "Agent": "DDPG"}, {"env_step": 1160000, "rew": 11843.9752, "rew_std": 752.8546, "Agent": "DDPG"}, {"env_step": 1165000, "rew": 12029.565, "rew_std": 598.7713, "Agent": "DDPG"}, {"env_step": 1170000, "rew": 11770.2589, "rew_std": 490.3858, "Agent": "DDPG"}, {"env_step": 1175000, "rew": 11822.2428, "rew_std": 784.5172, "Agent": "DDPG"}, {"env_step": 1180000, "rew": 11512.4653, "rew_std": 766.8959, "Agent": "DDPG"}, {"env_step": 1185000, "rew": 11956.7763, "rew_std": 456.4773, "Agent": "DDPG"}, {"env_step": 1190000, "rew": 11728.895, "rew_std": 673.0261, "Agent": "DDPG"}, {"env_step": 1195000, "rew": 11935.8341, "rew_std": 574.4006, "Agent": "DDPG"}, {"env_step": 1200000, "rew": 11926.2648, "rew_std": 355.5858, "Agent": "DDPG"}, {"env_step": 1205000, "rew": 11928.9478, "rew_std": 478.6533, "Agent": "DDPG"}, {"env_step": 1210000, "rew": 11856.7503, "rew_std": 700.5533, "Agent": "DDPG"}, {"env_step": 1215000, "rew": 12008.802, "rew_std": 387.26, "Agent": "DDPG"}, {"env_step": 1220000, "rew": 12019.8192, "rew_std": 289.377, "Agent": "DDPG"}, {"env_step": 1225000, "rew": 12057.7479, "rew_std": 414.885, "Agent": "DDPG"}, {"env_step": 1230000, "rew": 11917.911, "rew_std": 378.1731, "Agent": "DDPG"}, {"env_step": 1235000, "rew": 12072.7728, "rew_std": 642.2292, "Agent": "DDPG"}, {"env_step": 1240000, "rew": 11910.5075, "rew_std": 415.5465, "Agent": "DDPG"}, {"env_step": 1245000, "rew": 12119.5313, "rew_std": 376.6005, "Agent": "DDPG"}, {"env_step": 1250000, "rew": 11859.1451, "rew_std": 702.3344, "Agent": "DDPG"}, {"env_step": 5000, "rew": -491.7738, "rew_std": 108.7776, "Agent": "TD3"}, {"env_step": 10000, "rew": -262.0194, "rew_std": 291.8401, "Agent": "TD3"}, {"env_step": 15000, "rew": 939.3631, "rew_std": 527.7309, "Agent": "TD3"}, {"env_step": 20000, "rew": 1252.1263, "rew_std": 495.519, "Agent": "TD3"}, {"env_step": 25000, "rew": 2023.157, "rew_std": 523.8599, "Agent": "TD3"}, {"env_step": 30000, "rew": 2470.4407, "rew_std": 572.3322, "Agent": "TD3"}, {"env_step": 35000, "rew": 2749.4355, "rew_std": 598.9823, "Agent": "TD3"}, {"env_step": 40000, "rew": 3033.9985, "rew_std": 779.7287, "Agent": "TD3"}, {"env_step": 45000, "rew": 3414.1016, "rew_std": 557.1514, "Agent": "TD3"}, {"env_step": 50000, "rew": 3891.1476, "rew_std": 538.9045, "Agent": "TD3"}, {"env_step": 55000, "rew": 3769.953, "rew_std": 880.9102, "Agent": "TD3"}, {"env_step": 60000, "rew": 4215.0849, "rew_std": 338.3772, "Agent": "TD3"}, {"env_step": 65000, "rew": 4181.9487, "rew_std": 574.3375, "Agent": "TD3"}, {"env_step": 70000, "rew": 4451.898, "rew_std": 650.4167, "Agent": "TD3"}, {"env_step": 75000, "rew": 4715.3097, "rew_std": 387.332, "Agent": "TD3"}, {"env_step": 80000, "rew": 4977.8756, "rew_std": 332.4097, "Agent": "TD3"}, {"env_step": 85000, "rew": 4896.933, "rew_std": 436.9965, "Agent": "TD3"}, {"env_step": 90000, "rew": 5098.2884, "rew_std": 380.5583, "Agent": "TD3"}, {"env_step": 95000, "rew": 5064.4533, "rew_std": 376.0554, "Agent": "TD3"}, {"env_step": 100000, "rew": 5335.7776, "rew_std": 376.493, "Agent": "TD3"}, {"env_step": 105000, "rew": 5467.5474, "rew_std": 373.8211, "Agent": "TD3"}, {"env_step": 110000, "rew": 5548.1319, "rew_std": 376.2427, "Agent": "TD3"}, {"env_step": 115000, "rew": 5569.3028, "rew_std": 518.1977, "Agent": "TD3"}, {"env_step": 120000, "rew": 5557.8942, "rew_std": 501.4938, "Agent": "TD3"}, {"env_step": 125000, "rew": 5722.358, "rew_std": 351.7448, "Agent": "TD3"}, {"env_step": 130000, "rew": 5758.6115, "rew_std": 424.7812, "Agent": "TD3"}, {"env_step": 135000, "rew": 5848.0071, "rew_std": 496.162, "Agent": "TD3"}, {"env_step": 140000, "rew": 6062.7877, "rew_std": 456.7988, "Agent": "TD3"}, {"env_step": 145000, "rew": 5828.3979, "rew_std": 584.2805, "Agent": "TD3"}, {"env_step": 150000, "rew": 6110.8245, "rew_std": 399.7731, "Agent": "TD3"}, {"env_step": 155000, "rew": 6157.971, "rew_std": 451.4417, "Agent": "TD3"}, {"env_step": 160000, "rew": 6260.956, "rew_std": 455.5466, "Agent": "TD3"}, {"env_step": 165000, "rew": 6117.1841, "rew_std": 576.6519, "Agent": "TD3"}, {"env_step": 170000, "rew": 6360.7148, "rew_std": 465.9206, "Agent": "TD3"}, {"env_step": 175000, "rew": 6488.1046, "rew_std": 564.2199, "Agent": "TD3"}, {"env_step": 180000, "rew": 6485.5139, "rew_std": 523.2218, "Agent": "TD3"}, {"env_step": 185000, "rew": 6511.1075, "rew_std": 624.1234, "Agent": "TD3"}, {"env_step": 190000, "rew": 6623.9033, "rew_std": 606.1132, "Agent": "TD3"}, {"env_step": 195000, "rew": 6753.3993, "rew_std": 578.0539, "Agent": "TD3"}, {"env_step": 200000, "rew": 6720.6971, "rew_std": 617.1835, "Agent": "TD3"}, {"env_step": 205000, "rew": 6882.3305, "rew_std": 679.724, "Agent": "TD3"}, {"env_step": 210000, "rew": 6880.4545, "rew_std": 707.2802, "Agent": "TD3"}, {"env_step": 215000, "rew": 6899.3154, "rew_std": 623.297, "Agent": "TD3"}, {"env_step": 220000, "rew": 6959.4102, "rew_std": 532.1334, "Agent": "TD3"}, {"env_step": 225000, "rew": 6920.9057, "rew_std": 511.4183, "Agent": "TD3"}, {"env_step": 230000, "rew": 7056.547, "rew_std": 597.4038, "Agent": "TD3"}, {"env_step": 235000, "rew": 7163.5816, "rew_std": 656.1597, "Agent": "TD3"}, {"env_step": 240000, "rew": 7138.8121, "rew_std": 625.1124, "Agent": "TD3"}, {"env_step": 245000, "rew": 7201.8415, "rew_std": 736.7508, "Agent": "TD3"}, {"env_step": 250000, "rew": 7254.6046, "rew_std": 704.2153, "Agent": "TD3"}, {"env_step": 255000, "rew": 7104.3644, "rew_std": 774.2303, "Agent": "TD3"}, {"env_step": 260000, "rew": 7358.2901, "rew_std": 657.0822, "Agent": "TD3"}, {"env_step": 265000, "rew": 7387.0317, "rew_std": 690.7049, "Agent": "TD3"}, {"env_step": 270000, "rew": 7399.2832, "rew_std": 627.2569, "Agent": "TD3"}, {"env_step": 275000, "rew": 7461.1843, "rew_std": 752.2841, "Agent": "TD3"}, {"env_step": 280000, "rew": 7507.7466, "rew_std": 668.2431, "Agent": "TD3"}, {"env_step": 285000, "rew": 7564.6746, "rew_std": 704.8122, "Agent": "TD3"}, {"env_step": 290000, "rew": 7651.8881, "rew_std": 744.0028, "Agent": "TD3"}, {"env_step": 295000, "rew": 7639.7461, "rew_std": 683.6467, "Agent": "TD3"}, {"env_step": 300000, "rew": 7545.8266, "rew_std": 716.7928, "Agent": "TD3"}, {"env_step": 305000, "rew": 7586.793, "rew_std": 633.4799, "Agent": "TD3"}, {"env_step": 310000, "rew": 7830.0758, "rew_std": 786.95, "Agent": "TD3"}, {"env_step": 315000, "rew": 7753.983, "rew_std": 751.0059, "Agent": "TD3"}, {"env_step": 320000, "rew": 7838.6571, "rew_std": 738.9047, "Agent": "TD3"}, {"env_step": 325000, "rew": 7792.9696, "rew_std": 740.962, "Agent": "TD3"}, {"env_step": 330000, "rew": 7909.5051, "rew_std": 752.5249, "Agent": "TD3"}, {"env_step": 335000, "rew": 7940.4501, "rew_std": 803.4945, "Agent": "TD3"}, {"env_step": 340000, "rew": 7962.7489, "rew_std": 717.4368, "Agent": "TD3"}, {"env_step": 345000, "rew": 8013.5734, "rew_std": 914.0881, "Agent": "TD3"}, {"env_step": 350000, "rew": 8012.9812, "rew_std": 746.8912, "Agent": "TD3"}, {"env_step": 355000, "rew": 8070.066, "rew_std": 728.7675, "Agent": "TD3"}, {"env_step": 360000, "rew": 8029.628, "rew_std": 815.7537, "Agent": "TD3"}, {"env_step": 365000, "rew": 8189.133, "rew_std": 768.2708, "Agent": "TD3"}, {"env_step": 370000, "rew": 8138.5234, "rew_std": 870.01, "Agent": "TD3"}, {"env_step": 375000, "rew": 8199.8861, "rew_std": 749.7979, "Agent": "TD3"}, {"env_step": 380000, "rew": 8178.8484, "rew_std": 704.411, "Agent": "TD3"}, {"env_step": 385000, "rew": 8218.0795, "rew_std": 760.3696, "Agent": "TD3"}, {"env_step": 390000, "rew": 8320.7244, "rew_std": 742.0684, "Agent": "TD3"}, {"env_step": 395000, "rew": 8306.1744, "rew_std": 774.4848, "Agent": "TD3"}, {"env_step": 400000, "rew": 8332.8264, "rew_std": 662.934, "Agent": "TD3"}, {"env_step": 405000, "rew": 8454.6759, "rew_std": 784.0574, "Agent": "TD3"}, {"env_step": 410000, "rew": 8435.5763, "rew_std": 857.1755, "Agent": "TD3"}, {"env_step": 415000, "rew": 8429.4245, "rew_std": 696.4486, "Agent": "TD3"}, {"env_step": 420000, "rew": 8507.612, "rew_std": 759.0253, "Agent": "TD3"}, {"env_step": 425000, "rew": 8407.1033, "rew_std": 731.4275, "Agent": "TD3"}, {"env_step": 430000, "rew": 8494.8885, "rew_std": 743.2674, "Agent": "TD3"}, {"env_step": 435000, "rew": 8612.279, "rew_std": 793.3193, "Agent": "TD3"}, {"env_step": 440000, "rew": 8607.9371, "rew_std": 747.9842, "Agent": "TD3"}, {"env_step": 445000, "rew": 8591.4923, "rew_std": 747.7889, "Agent": "TD3"}, {"env_step": 450000, "rew": 8595.9424, "rew_std": 824.3331, "Agent": "TD3"}, {"env_step": 455000, "rew": 8677.8482, "rew_std": 772.3754, "Agent": "TD3"}, {"env_step": 460000, "rew": 8598.7622, "rew_std": 820.0984, "Agent": "TD3"}, {"env_step": 465000, "rew": 8559.2102, "rew_std": 880.4971, "Agent": "TD3"}, {"env_step": 470000, "rew": 8745.6001, "rew_std": 739.5832, "Agent": "TD3"}, {"env_step": 475000, "rew": 8737.3063, "rew_std": 747.7016, "Agent": "TD3"}, {"env_step": 480000, "rew": 8790.2033, "rew_std": 752.6937, "Agent": "TD3"}, {"env_step": 485000, "rew": 8760.4856, "rew_std": 774.4096, "Agent": "TD3"}, {"env_step": 490000, "rew": 8824.4661, "rew_std": 818.5167, "Agent": "TD3"}, {"env_step": 495000, "rew": 8831.1715, "rew_std": 787.5662, "Agent": "TD3"}, {"env_step": 500000, "rew": 8804.9793, "rew_std": 731.6913, "Agent": "TD3"}, {"env_step": 505000, "rew": 8850.2683, "rew_std": 735.7623, "Agent": "TD3"}, {"env_step": 510000, "rew": 8760.5785, "rew_std": 871.1392, "Agent": "TD3"}, {"env_step": 515000, "rew": 8909.451, "rew_std": 750.5147, "Agent": "TD3"}, {"env_step": 520000, "rew": 8898.6247, "rew_std": 823.3533, "Agent": "TD3"}, {"env_step": 525000, "rew": 8979.796, "rew_std": 803.2727, "Agent": "TD3"}, {"env_step": 530000, "rew": 8826.3866, "rew_std": 688.6222, "Agent": "TD3"}, {"env_step": 535000, "rew": 8943.3122, "rew_std": 862.1328, "Agent": "TD3"}, {"env_step": 540000, "rew": 8950.6791, "rew_std": 841.9943, "Agent": "TD3"}, {"env_step": 545000, "rew": 9076.3193, "rew_std": 807.2128, "Agent": "TD3"}, {"env_step": 550000, "rew": 9083.1078, "rew_std": 783.0794, "Agent": "TD3"}, {"env_step": 555000, "rew": 9100.463, "rew_std": 788.4377, "Agent": "TD3"}, {"env_step": 560000, "rew": 9109.1642, "rew_std": 766.6449, "Agent": "TD3"}, {"env_step": 565000, "rew": 9035.3107, "rew_std": 832.0449, "Agent": "TD3"}, {"env_step": 570000, "rew": 9088.8057, "rew_std": 692.5742, "Agent": "TD3"}, {"env_step": 575000, "rew": 9071.9369, "rew_std": 754.8484, "Agent": "TD3"}, {"env_step": 580000, "rew": 9110.8467, "rew_std": 743.0409, "Agent": "TD3"}, {"env_step": 585000, "rew": 9193.2445, "rew_std": 875.4127, "Agent": "TD3"}, {"env_step": 590000, "rew": 9195.2836, "rew_std": 813.2878, "Agent": "TD3"}, {"env_step": 595000, "rew": 9187.9049, "rew_std": 740.0919, "Agent": "TD3"}, {"env_step": 600000, "rew": 9327.7404, "rew_std": 806.1976, "Agent": "TD3"}, {"env_step": 605000, "rew": 9138.3398, "rew_std": 725.0365, "Agent": "TD3"}, {"env_step": 610000, "rew": 9264.9501, "rew_std": 799.4507, "Agent": "TD3"}, {"env_step": 615000, "rew": 9312.8531, "rew_std": 837.1041, "Agent": "TD3"}, {"env_step": 620000, "rew": 9283.921, "rew_std": 826.1715, "Agent": "TD3"}, {"env_step": 625000, "rew": 9361.8274, "rew_std": 825.3747, "Agent": "TD3"}, {"env_step": 630000, "rew": 9282.1575, "rew_std": 730.256, "Agent": "TD3"}, {"env_step": 635000, "rew": 9377.9864, "rew_std": 793.3509, "Agent": "TD3"}, {"env_step": 640000, "rew": 9284.64, "rew_std": 737.4991, "Agent": "TD3"}, {"env_step": 645000, "rew": 9341.4001, "rew_std": 796.181, "Agent": "TD3"}, {"env_step": 650000, "rew": 9393.1446, "rew_std": 719.3792, "Agent": "TD3"}, {"env_step": 655000, "rew": 9431.7345, "rew_std": 797.8404, "Agent": "TD3"}, {"env_step": 660000, "rew": 9517.6054, "rew_std": 840.7985, "Agent": "TD3"}, {"env_step": 665000, "rew": 9404.8517, "rew_std": 747.3251, "Agent": "TD3"}, {"env_step": 670000, "rew": 9454.8159, "rew_std": 760.1696, "Agent": "TD3"}, {"env_step": 675000, "rew": 9500.148, "rew_std": 787.3883, "Agent": "TD3"}, {"env_step": 680000, "rew": 9561.3347, "rew_std": 785.2622, "Agent": "TD3"}, {"env_step": 685000, "rew": 9487.3321, "rew_std": 818.1828, "Agent": "TD3"}, {"env_step": 690000, "rew": 9426.4945, "rew_std": 792.3134, "Agent": "TD3"}, {"env_step": 695000, "rew": 9489.5022, "rew_std": 765.5105, "Agent": "TD3"}, {"env_step": 700000, "rew": 9341.9792, "rew_std": 846.4012, "Agent": "TD3"}, {"env_step": 705000, "rew": 9454.9199, "rew_std": 764.8516, "Agent": "TD3"}, {"env_step": 710000, "rew": 9589.4149, "rew_std": 772.9486, "Agent": "TD3"}, {"env_step": 715000, "rew": 9645.4927, "rew_std": 780.9173, "Agent": "TD3"}, {"env_step": 720000, "rew": 9668.0014, "rew_std": 785.7856, "Agent": "TD3"}, {"env_step": 725000, "rew": 9625.1669, "rew_std": 737.8234, "Agent": "TD3"}, {"env_step": 730000, "rew": 9502.0475, "rew_std": 968.5391, "Agent": "TD3"}, {"env_step": 735000, "rew": 9483.871, "rew_std": 690.854, "Agent": "TD3"}, {"env_step": 740000, "rew": 9641.7172, "rew_std": 793.3571, "Agent": "TD3"}, {"env_step": 745000, "rew": 9734.494, "rew_std": 812.6246, "Agent": "TD3"}, {"env_step": 750000, "rew": 9720.93, "rew_std": 783.6363, "Agent": "TD3"}, {"env_step": 755000, "rew": 9665.8567, "rew_std": 815.7909, "Agent": "TD3"}, {"env_step": 760000, "rew": 9679.3486, "rew_std": 800.5279, "Agent": "TD3"}, {"env_step": 765000, "rew": 9763.7048, "rew_std": 834.6644, "Agent": "TD3"}, {"env_step": 770000, "rew": 9772.4914, "rew_std": 832.7718, "Agent": "TD3"}, {"env_step": 775000, "rew": 9804.0023, "rew_std": 792.9635, "Agent": "TD3"}, {"env_step": 780000, "rew": 9745.1979, "rew_std": 920.7406, "Agent": "TD3"}, {"env_step": 785000, "rew": 9750.7988, "rew_std": 784.167, "Agent": "TD3"}, {"env_step": 790000, "rew": 9808.6392, "rew_std": 822.0098, "Agent": "TD3"}, {"env_step": 795000, "rew": 9754.6227, "rew_std": 831.5913, "Agent": "TD3"}, {"env_step": 800000, "rew": 9778.1098, "rew_std": 763.7581, "Agent": "TD3"}, {"env_step": 805000, "rew": 9825.7902, "rew_std": 813.3548, "Agent": "TD3"}, {"env_step": 810000, "rew": 9798.4076, "rew_std": 799.6761, "Agent": "TD3"}, {"env_step": 815000, "rew": 9804.338, "rew_std": 798.6376, "Agent": "TD3"}, {"env_step": 820000, "rew": 9793.0963, "rew_std": 809.0504, "Agent": "TD3"}, {"env_step": 825000, "rew": 9821.5347, "rew_std": 811.4409, "Agent": "TD3"}, {"env_step": 830000, "rew": 9762.1984, "rew_std": 753.4929, "Agent": "TD3"}, {"env_step": 835000, "rew": 9852.6895, "rew_std": 841.7706, "Agent": "TD3"}, {"env_step": 840000, "rew": 9872.1575, "rew_std": 756.2683, "Agent": "TD3"}, {"env_step": 845000, "rew": 9909.5908, "rew_std": 837.792, "Agent": "TD3"}, {"env_step": 850000, "rew": 9916.1449, "rew_std": 807.2152, "Agent": "TD3"}, {"env_step": 855000, "rew": 9883.2548, "rew_std": 776.8632, "Agent": "TD3"}, {"env_step": 860000, "rew": 9974.5868, "rew_std": 808.4498, "Agent": "TD3"}, {"env_step": 865000, "rew": 9928.346, "rew_std": 811.971, "Agent": "TD3"}, {"env_step": 870000, "rew": 9910.8146, "rew_std": 829.7343, "Agent": "TD3"}, {"env_step": 875000, "rew": 9915.3258, "rew_std": 716.3259, "Agent": "TD3"}, {"env_step": 880000, "rew": 9943.3313, "rew_std": 801.7854, "Agent": "TD3"}, {"env_step": 885000, "rew": 9973.9117, "rew_std": 815.8196, "Agent": "TD3"}, {"env_step": 890000, "rew": 10028.523, "rew_std": 810.603, "Agent": "TD3"}, {"env_step": 895000, "rew": 9988.4478, "rew_std": 830.4222, "Agent": "TD3"}, {"env_step": 900000, "rew": 10019.7384, "rew_std": 821.4592, "Agent": "TD3"}, {"env_step": 905000, "rew": 10014.3362, "rew_std": 800.811, "Agent": "TD3"}, {"env_step": 910000, "rew": 10115.0379, "rew_std": 802.7237, "Agent": "TD3"}, {"env_step": 915000, "rew": 10078.2624, "rew_std": 763.4506, "Agent": "TD3"}, {"env_step": 920000, "rew": 10025.5229, "rew_std": 706.419, "Agent": "TD3"}, {"env_step": 925000, "rew": 10137.1765, "rew_std": 784.5741, "Agent": "TD3"}, {"env_step": 930000, "rew": 10029.2503, "rew_std": 786.896, "Agent": "TD3"}, {"env_step": 935000, "rew": 10059.9907, "rew_std": 770.6038, "Agent": "TD3"}, {"env_step": 940000, "rew": 10080.8968, "rew_std": 868.2282, "Agent": "TD3"}, {"env_step": 945000, "rew": 10087.6062, "rew_std": 761.3788, "Agent": "TD3"}, {"env_step": 950000, "rew": 10130.8133, "rew_std": 817.9981, "Agent": "TD3"}, {"env_step": 955000, "rew": 10119.0627, "rew_std": 797.4048, "Agent": "TD3"}, {"env_step": 960000, "rew": 10117.4533, "rew_std": 731.4008, "Agent": "TD3"}, {"env_step": 965000, "rew": 10095.9303, "rew_std": 840.427, "Agent": "TD3"}, {"env_step": 970000, "rew": 10065.6145, "rew_std": 821.9565, "Agent": "TD3"}, {"env_step": 975000, "rew": 10168.2197, "rew_std": 772.3371, "Agent": "TD3"}, {"env_step": 980000, "rew": 10138.8102, "rew_std": 797.7195, "Agent": "TD3"}, {"env_step": 985000, "rew": 10201.1953, "rew_std": 772.7887, "Agent": "TD3"}, {"env_step": 990000, "rew": 10164.0714, "rew_std": 772.0474, "Agent": "TD3"}, {"env_step": 995000, "rew": 10166.9941, "rew_std": 782.1563, "Agent": "TD3"}, {"env_step": 1000000, "rew": 10126.7338, "rew_std": 691.8358, "Agent": "TD3"}, {"env_step": 1005000, "rew": 10178.6879, "rew_std": 730.893, "Agent": "TD3"}, {"env_step": 1010000, "rew": 10218.8769, "rew_std": 779.1047, "Agent": "TD3"}, {"env_step": 1015000, "rew": 10223.7908, "rew_std": 805.2236, "Agent": "TD3"}, {"env_step": 1020000, "rew": 10283.0535, "rew_std": 840.6199, "Agent": "TD3"}, {"env_step": 1025000, "rew": 10234.8471, "rew_std": 737.6595, "Agent": "TD3"}, {"env_step": 1030000, "rew": 10276.3837, "rew_std": 851.1605, "Agent": "TD3"}, {"env_step": 1035000, "rew": 10326.8339, "rew_std": 794.7202, "Agent": "TD3"}, {"env_step": 1040000, "rew": 10296.1702, "rew_std": 756.8565, "Agent": "TD3"}, {"env_step": 1045000, "rew": 10303.7155, "rew_std": 793.3612, "Agent": "TD3"}, {"env_step": 1050000, "rew": 10323.1621, "rew_std": 822.8054, "Agent": "TD3"}, {"env_step": 1055000, "rew": 10317.0148, "rew_std": 815.1317, "Agent": "TD3"}, {"env_step": 1060000, "rew": 10384.3829, "rew_std": 792.7379, "Agent": "TD3"}, {"env_step": 1065000, "rew": 10312.6868, "rew_std": 802.5368, "Agent": "TD3"}, {"env_step": 1070000, "rew": 10337.5452, "rew_std": 731.5095, "Agent": "TD3"}, {"env_step": 1075000, "rew": 10426.9081, "rew_std": 720.6897, "Agent": "TD3"}, {"env_step": 1080000, "rew": 10445.9585, "rew_std": 743.8478, "Agent": "TD3"}, {"env_step": 1085000, "rew": 10404.4356, "rew_std": 686.9511, "Agent": "TD3"}, {"env_step": 1090000, "rew": 10411.9082, "rew_std": 759.3062, "Agent": "TD3"}, {"env_step": 1095000, "rew": 10493.6734, "rew_std": 719.0233, "Agent": "TD3"}, {"env_step": 1100000, "rew": 10446.2659, "rew_std": 800.8762, "Agent": "TD3"}, {"env_step": 1105000, "rew": 10473.0588, "rew_std": 714.3254, "Agent": "TD3"}, {"env_step": 1110000, "rew": 10559.5191, "rew_std": 748.8673, "Agent": "TD3"}, {"env_step": 1115000, "rew": 10549.2806, "rew_std": 754.3475, "Agent": "TD3"}, {"env_step": 1120000, "rew": 10452.5229, "rew_std": 692.153, "Agent": "TD3"}, {"env_step": 1125000, "rew": 10562.384, "rew_std": 770.6165, "Agent": "TD3"}, {"env_step": 1130000, "rew": 10547.1885, "rew_std": 711.6438, "Agent": "TD3"}, {"env_step": 1135000, "rew": 10602.5228, "rew_std": 721.8553, "Agent": "TD3"}, {"env_step": 1140000, "rew": 10459.9711, "rew_std": 688.9873, "Agent": "TD3"}, {"env_step": 1145000, "rew": 10592.8877, "rew_std": 706.5442, "Agent": "TD3"}, {"env_step": 1150000, "rew": 10587.6289, "rew_std": 766.3566, "Agent": "TD3"}, {"env_step": 1155000, "rew": 10635.7845, "rew_std": 774.0688, "Agent": "TD3"}, {"env_step": 1160000, "rew": 10659.2155, "rew_std": 774.1428, "Agent": "TD3"}, {"env_step": 1165000, "rew": 10574.2337, "rew_std": 678.7509, "Agent": "TD3"}, {"env_step": 1170000, "rew": 10614.9555, "rew_std": 700.4832, "Agent": "TD3"}, {"env_step": 1175000, "rew": 10676.1137, "rew_std": 676.5039, "Agent": "TD3"}, {"env_step": 1180000, "rew": 10698.8282, "rew_std": 658.7235, "Agent": "TD3"}, {"env_step": 1185000, "rew": 10542.7353, "rew_std": 663.1433, "Agent": "TD3"}, {"env_step": 1190000, "rew": 10669.2207, "rew_std": 709.0098, "Agent": "TD3"}, {"env_step": 1195000, "rew": 10655.4834, "rew_std": 698.5363, "Agent": "TD3"}, {"env_step": 1200000, "rew": 10633.3746, "rew_std": 638.7168, "Agent": "TD3"}, {"env_step": 1205000, "rew": 10701.2624, "rew_std": 710.2126, "Agent": "TD3"}, {"env_step": 1210000, "rew": 10764.1783, "rew_std": 624.228, "Agent": "TD3"}, {"env_step": 1215000, "rew": 10750.3966, "rew_std": 695.4035, "Agent": "TD3"}, {"env_step": 1220000, "rew": 10837.0146, "rew_std": 677.4937, "Agent": "TD3"}, {"env_step": 1225000, "rew": 10737.5749, "rew_std": 714.6048, "Agent": "TD3"}, {"env_step": 1230000, "rew": 10771.3394, "rew_std": 627.9872, "Agent": "TD3"}, {"env_step": 1235000, "rew": 10858.5293, "rew_std": 689.6976, "Agent": "TD3"}, {"env_step": 1240000, "rew": 10797.6929, "rew_std": 668.7104, "Agent": "TD3"}, {"env_step": 1245000, "rew": 10868.2019, "rew_std": 696.346, "Agent": "TD3"}, {"env_step": 1250000, "rew": 10792.3989, "rew_std": 665.2522, "Agent": "TD3"}, {"env_step": 0, "rew": -46.1947, "rew_std": 25.5804, "Agent": "TRPO"}, {"env_step": 30720, "rew": -13.9012, "rew_std": 11.913, "Agent": "TRPO"}, {"env_step": 61440, "rew": 18.0615, "rew_std": 9.0166, "Agent": "TRPO"}, {"env_step": 92160, "rew": 51.9175, "rew_std": 16.4955, "Agent": "TRPO"}, {"env_step": 122880, "rew": 108.4958, "rew_std": 33.2241, "Agent": "TRPO"}, {"env_step": 153600, "rew": 183.9944, "rew_std": 35.8262, "Agent": "TRPO"}, {"env_step": 184320, "rew": 257.4861, "rew_std": 60.051, "Agent": "TRPO"}, {"env_step": 215040, "rew": 430.4068, "rew_std": 124.2541, "Agent": "TRPO"}, {"env_step": 245760, "rew": 564.2017, "rew_std": 146.0606, "Agent": "TRPO"}, {"env_step": 276480, "rew": 682.5168, "rew_std": 122.3187, "Agent": "TRPO"}, {"env_step": 307200, "rew": 832.5033, "rew_std": 159.2968, "Agent": "TRPO"}, {"env_step": 337920, "rew": 1014.1049, "rew_std": 221.8108, "Agent": "TRPO"}, {"env_step": 368640, "rew": 1128.5618, "rew_std": 167.9213, "Agent": "TRPO"}, {"env_step": 399360, "rew": 1146.4494, "rew_std": 215.1602, "Agent": "TRPO"}, {"env_step": 430080, "rew": 1233.6374, "rew_std": 286.8604, "Agent": "TRPO"}, {"env_step": 460800, "rew": 1261.4876, "rew_std": 301.0892, "Agent": "TRPO"}, {"env_step": 491520, "rew": 1472.6865, "rew_std": 215.7148, "Agent": "TRPO"}, {"env_step": 522240, "rew": 1501.5343, "rew_std": 428.1053, "Agent": "TRPO"}, {"env_step": 552960, "rew": 1631.5161, "rew_std": 558.1692, "Agent": "TRPO"}, {"env_step": 583680, "rew": 1741.6005, "rew_std": 483.7599, "Agent": "TRPO"}, {"env_step": 614400, "rew": 1809.3667, "rew_std": 444.8329, "Agent": "TRPO"}, {"env_step": 645120, "rew": 1943.2351, "rew_std": 386.0455, "Agent": "TRPO"}, {"env_step": 675840, "rew": 2119.3593, "rew_std": 463.9853, "Agent": "TRPO"}, {"env_step": 706560, "rew": 2160.9349, "rew_std": 299.8534, "Agent": "TRPO"}, {"env_step": 737280, "rew": 2205.6584, "rew_std": 495.8226, "Agent": "TRPO"}, {"env_step": 768000, "rew": 2148.4616, "rew_std": 617.4263, "Agent": "TRPO"}, {"env_step": 798720, "rew": 2294.2347, "rew_std": 619.654, "Agent": "TRPO"}, {"env_step": 829440, "rew": 2334.3877, "rew_std": 569.8366, "Agent": "TRPO"}, {"env_step": 860160, "rew": 2373.0195, "rew_std": 635.4872, "Agent": "TRPO"}, {"env_step": 890880, "rew": 2572.1874, "rew_std": 771.2781, "Agent": "TRPO"}, {"env_step": 921600, "rew": 2568.2581, "rew_std": 539.7726, "Agent": "TRPO"}, {"env_step": 952320, "rew": 2639.2867, "rew_std": 572.4301, "Agent": "TRPO"}, {"env_step": 983040, "rew": 2866.6508, "rew_std": 707.8937, "Agent": "TRPO"}, {"env_step": 1013760, "rew": 2731.0697, "rew_std": 675.644, "Agent": "TRPO"}, {"env_step": 1044480, "rew": 2702.1965, "rew_std": 742.4807, "Agent": "TRPO"}, {"env_step": 1075200, "rew": 3001.7621, "rew_std": 688.8314, "Agent": "TRPO"}, {"env_step": 1105920, "rew": 2877.8716, "rew_std": 922.2411, "Agent": "TRPO"}, {"env_step": 1136640, "rew": 3104.9072, "rew_std": 800.8227, "Agent": "TRPO"}, {"env_step": 1167360, "rew": 2976.5528, "rew_std": 661.5153, "Agent": "TRPO"}, {"env_step": 1198080, "rew": 3196.2228, "rew_std": 707.9679, "Agent": "TRPO"}, {"env_step": 1228800, "rew": 3237.2005, "rew_std": 505.6685, "Agent": "TRPO"}, {"env_step": 1259520, "rew": 3098.3083, "rew_std": 578.4346, "Agent": "TRPO"}, {"env_step": 1290240, "rew": 3356.962, "rew_std": 786.3838, "Agent": "TRPO"}, {"env_step": 1320960, "rew": 3241.2647, "rew_std": 600.3495, "Agent": "TRPO"}, {"env_step": 1351680, "rew": 3361.1722, "rew_std": 547.5151, "Agent": "TRPO"}, {"env_step": 1382400, "rew": 3495.3879, "rew_std": 662.5387, "Agent": "TRPO"}, {"env_step": 1413120, "rew": 3388.8443, "rew_std": 537.1016, "Agent": "TRPO"}, {"env_step": 1443840, "rew": 3493.6456, "rew_std": 509.2391, "Agent": "TRPO"}, {"env_step": 1474560, "rew": 3256.3625, "rew_std": 412.1903, "Agent": "TRPO"}, {"env_step": 1505280, "rew": 3695.3862, "rew_std": 644.5284, "Agent": "TRPO"}, {"env_step": 1536000, "rew": 3685.0253, "rew_std": 537.4812, "Agent": "TRPO"}, {"env_step": 1566720, "rew": 3565.3736, "rew_std": 647.2313, "Agent": "TRPO"}, {"env_step": 1597440, "rew": 3790.5153, "rew_std": 702.1943, "Agent": "TRPO"}, {"env_step": 1628160, "rew": 3598.3267, "rew_std": 626.2541, "Agent": "TRPO"}, {"env_step": 1658880, "rew": 3970.7287, "rew_std": 647.9836, "Agent": "TRPO"}, {"env_step": 1689600, "rew": 3658.6492, "rew_std": 645.0697, "Agent": "TRPO"}, {"env_step": 1720320, "rew": 3848.0345, "rew_std": 701.1273, "Agent": "TRPO"}, {"env_step": 1751040, "rew": 4061.4513, "rew_std": 690.0289, "Agent": "TRPO"}, {"env_step": 1781760, "rew": 3807.921, "rew_std": 706.5999, "Agent": "TRPO"}, {"env_step": 1812480, "rew": 3802.0342, "rew_std": 663.7139, "Agent": "TRPO"}, {"env_step": 1843200, "rew": 3873.6662, "rew_std": 566.3833, "Agent": "TRPO"}, {"env_step": 1873920, "rew": 4038.6082, "rew_std": 518.5079, "Agent": "TRPO"}, {"env_step": 1904640, "rew": 4167.3221, "rew_std": 440.9658, "Agent": "TRPO"}, {"env_step": 1935360, "rew": 3922.2855, "rew_std": 733.8238, "Agent": "TRPO"}, {"env_step": 1966080, "rew": 4174.7307, "rew_std": 495.1849, "Agent": "TRPO"}, {"env_step": 1996800, "rew": 4155.8384, "rew_std": 477.6376, "Agent": "TRPO"}, {"env_step": 2027520, "rew": 4259.3055, "rew_std": 577.3636, "Agent": "TRPO"}, {"env_step": 2058240, "rew": 4256.1933, "rew_std": 594.2826, "Agent": "TRPO"}, {"env_step": 2088960, "rew": 4355.0669, "rew_std": 605.8978, "Agent": "TRPO"}, {"env_step": 2119680, "rew": 4297.2252, "rew_std": 474.421, "Agent": "TRPO"}, {"env_step": 2150400, "rew": 4287.4698, "rew_std": 363.7332, "Agent": "TRPO"}, {"env_step": 2181120, "rew": 4398.2137, "rew_std": 354.3603, "Agent": "TRPO"}, {"env_step": 2211840, "rew": 4413.5796, "rew_std": 460.9681, "Agent": "TRPO"}, {"env_step": 2242560, "rew": 4329.001, "rew_std": 530.2755, "Agent": "TRPO"}, {"env_step": 2273280, "rew": 4252.3464, "rew_std": 597.5882, "Agent": "TRPO"}, {"env_step": 2304000, "rew": 4462.5657, "rew_std": 495.3802, "Agent": "TRPO"}, {"env_step": 2334720, "rew": 4374.2838, "rew_std": 457.0956, "Agent": "TRPO"}, {"env_step": 2365440, "rew": 4529.4549, "rew_std": 461.1733, "Agent": "TRPO"}, {"env_step": 2396160, "rew": 4291.4468, "rew_std": 411.2142, "Agent": "TRPO"}, {"env_step": 2426880, "rew": 4430.1459, "rew_std": 601.8812, "Agent": "TRPO"}, {"env_step": 2457600, "rew": 4508.078, "rew_std": 465.0507, "Agent": "TRPO"}, {"env_step": 2488320, "rew": 4151.3796, "rew_std": 414.656, "Agent": "TRPO"}, {"env_step": 2519040, "rew": 4705.7212, "rew_std": 477.1206, "Agent": "TRPO"}, {"env_step": 2549760, "rew": 4641.3111, "rew_std": 622.6369, "Agent": "TRPO"}, {"env_step": 2580480, "rew": 4484.1452, "rew_std": 442.0811, "Agent": "TRPO"}, {"env_step": 2611200, "rew": 4658.7294, "rew_std": 432.1496, "Agent": "TRPO"}, {"env_step": 2641920, "rew": 4625.8785, "rew_std": 357.4208, "Agent": "TRPO"}, {"env_step": 2672640, "rew": 4284.9794, "rew_std": 599.7262, "Agent": "TRPO"}, {"env_step": 2703360, "rew": 4419.1972, "rew_std": 447.5325, "Agent": "TRPO"}, {"env_step": 2734080, "rew": 4746.1067, "rew_std": 580.4515, "Agent": "TRPO"}, {"env_step": 2764800, "rew": 4756.6644, "rew_std": 502.4658, "Agent": "TRPO"}, {"env_step": 2795520, "rew": 4607.7287, "rew_std": 427.1984, "Agent": "TRPO"}, {"env_step": 2826240, "rew": 4404.4363, "rew_std": 555.3297, "Agent": "TRPO"}, {"env_step": 2856960, "rew": 4816.8623, "rew_std": 528.6794, "Agent": "TRPO"}, {"env_step": 2887680, "rew": 4753.3578, "rew_std": 715.7057, "Agent": "TRPO"}, {"env_step": 2918400, "rew": 4604.763, "rew_std": 447.4458, "Agent": "TRPO"}, {"env_step": 2949120, "rew": 4800.2693, "rew_std": 602.9667, "Agent": "TRPO"}, {"env_step": 2979840, "rew": 4772.9396, "rew_std": 368.1691, "Agent": "TRPO"}, {"env_step": 3010560, "rew": 4897.0257, "rew_std": 449.1329, "Agent": "TRPO"}, {"env_step": 3041280, "rew": 4841.7814, "rew_std": 534.8951, "Agent": "TRPO"}, {"env_step": 3072000, "rew": 4604.9043, "rew_std": 580.976, "Agent": "TRPO"}, {"env_step": 0, "rew": -46.1947, "rew_std": 25.5804, "Agent": "NPG"}, {"env_step": 30720, "rew": -29.0009, "rew_std": 15.8916, "Agent": "NPG"}, {"env_step": 61440, "rew": -10.4279, "rew_std": 11.0242, "Agent": "NPG"}, {"env_step": 92160, "rew": 11.1187, "rew_std": 9.0737, "Agent": "NPG"}, {"env_step": 122880, "rew": 27.5852, "rew_std": 16.4638, "Agent": "NPG"}, {"env_step": 153600, "rew": 56.7285, "rew_std": 23.8642, "Agent": "NPG"}, {"env_step": 184320, "rew": 81.2319, "rew_std": 29.1525, "Agent": "NPG"}, {"env_step": 215040, "rew": 159.4704, "rew_std": 47.3793, "Agent": "NPG"}, {"env_step": 245760, "rew": 174.9781, "rew_std": 57.693, "Agent": "NPG"}, {"env_step": 276480, "rew": 178.6358, "rew_std": 32.807, "Agent": "NPG"}, {"env_step": 307200, "rew": 218.3212, "rew_std": 41.035, "Agent": "NPG"}, {"env_step": 337920, "rew": 368.663, "rew_std": 80.0623, "Agent": "NPG"}, {"env_step": 368640, "rew": 432.2163, "rew_std": 63.1225, "Agent": "NPG"}, {"env_step": 399360, "rew": 568.0807, "rew_std": 125.517, "Agent": "NPG"}, {"env_step": 430080, "rew": 631.1259, "rew_std": 114.6371, "Agent": "NPG"}, {"env_step": 460800, "rew": 753.5929, "rew_std": 118.5185, "Agent": "NPG"}, {"env_step": 491520, "rew": 864.7371, "rew_std": 183.4071, "Agent": "NPG"}, {"env_step": 522240, "rew": 940.2978, "rew_std": 158.8018, "Agent": "NPG"}, {"env_step": 552960, "rew": 1021.7661, "rew_std": 159.2406, "Agent": "NPG"}, {"env_step": 583680, "rew": 1131.714, "rew_std": 150.4756, "Agent": "NPG"}, {"env_step": 614400, "rew": 1201.5181, "rew_std": 164.258, "Agent": "NPG"}, {"env_step": 645120, "rew": 1343.1191, "rew_std": 184.6686, "Agent": "NPG"}, {"env_step": 675840, "rew": 1318.401, "rew_std": 229.7468, "Agent": "NPG"}, {"env_step": 706560, "rew": 1459.1159, "rew_std": 222.6298, "Agent": "NPG"}, {"env_step": 737280, "rew": 1542.0295, "rew_std": 203.9479, "Agent": "NPG"}, {"env_step": 768000, "rew": 1634.9176, "rew_std": 218.2131, "Agent": "NPG"}, {"env_step": 798720, "rew": 1692.0983, "rew_std": 343.9785, "Agent": "NPG"}, {"env_step": 829440, "rew": 1906.1507, "rew_std": 457.5102, "Agent": "NPG"}, {"env_step": 860160, "rew": 1775.1748, "rew_std": 342.0144, "Agent": "NPG"}, {"env_step": 890880, "rew": 2102.0494, "rew_std": 355.8503, "Agent": "NPG"}, {"env_step": 921600, "rew": 2046.7794, "rew_std": 340.8884, "Agent": "NPG"}, {"env_step": 952320, "rew": 2164.3512, "rew_std": 405.7482, "Agent": "NPG"}, {"env_step": 983040, "rew": 2348.312, "rew_std": 509.9401, "Agent": "NPG"}, {"env_step": 1013760, "rew": 2365.8958, "rew_std": 523.6603, "Agent": "NPG"}, {"env_step": 1044480, "rew": 2324.9164, "rew_std": 528.4526, "Agent": "NPG"}, {"env_step": 1075200, "rew": 2493.722, "rew_std": 583.8895, "Agent": "NPG"}, {"env_step": 1105920, "rew": 2638.7282, "rew_std": 650.3356, "Agent": "NPG"}, {"env_step": 1136640, "rew": 2588.3598, "rew_std": 506.4239, "Agent": "NPG"}, {"env_step": 1167360, "rew": 2657.8218, "rew_std": 542.1361, "Agent": "NPG"}, {"env_step": 1198080, "rew": 2772.0887, "rew_std": 399.8722, "Agent": "NPG"}, {"env_step": 1228800, "rew": 2983.8532, "rew_std": 405.1427, "Agent": "NPG"}, {"env_step": 1259520, "rew": 2969.912, "rew_std": 543.0224, "Agent": "NPG"}, {"env_step": 1290240, "rew": 2898.5856, "rew_std": 687.1927, "Agent": "NPG"}, {"env_step": 1320960, "rew": 2979.687, "rew_std": 685.3181, "Agent": "NPG"}, {"env_step": 1351680, "rew": 2937.7865, "rew_std": 654.2811, "Agent": "NPG"}, {"env_step": 1382400, "rew": 3180.3262, "rew_std": 502.1675, "Agent": "NPG"}, {"env_step": 1413120, "rew": 3181.4746, "rew_std": 526.9918, "Agent": "NPG"}, {"env_step": 1443840, "rew": 3186.725, "rew_std": 483.0104, "Agent": "NPG"}, {"env_step": 1474560, "rew": 3347.1624, "rew_std": 474.3212, "Agent": "NPG"}, {"env_step": 1505280, "rew": 3264.1247, "rew_std": 542.1326, "Agent": "NPG"}, {"env_step": 1536000, "rew": 3267.3737, "rew_std": 536.477, "Agent": "NPG"}, {"env_step": 1566720, "rew": 3677.709, "rew_std": 486.0857, "Agent": "NPG"}, {"env_step": 1597440, "rew": 3379.1083, "rew_std": 683.1763, "Agent": "NPG"}, {"env_step": 1628160, "rew": 3533.6653, "rew_std": 437.9623, "Agent": "NPG"}, {"env_step": 1658880, "rew": 3534.0292, "rew_std": 787.6867, "Agent": "NPG"}, {"env_step": 1689600, "rew": 3399.7589, "rew_std": 685.1419, "Agent": "NPG"}, {"env_step": 1720320, "rew": 3587.7024, "rew_std": 471.134, "Agent": "NPG"}, {"env_step": 1751040, "rew": 3560.6233, "rew_std": 616.8093, "Agent": "NPG"}, {"env_step": 1781760, "rew": 3635.7303, "rew_std": 608.4666, "Agent": "NPG"}, {"env_step": 1812480, "rew": 3713.5899, "rew_std": 802.8577, "Agent": "NPG"}, {"env_step": 1843200, "rew": 3590.908, "rew_std": 557.7779, "Agent": "NPG"}, {"env_step": 1873920, "rew": 3903.7812, "rew_std": 655.7805, "Agent": "NPG"}, {"env_step": 1904640, "rew": 3770.5075, "rew_std": 645.9229, "Agent": "NPG"}, {"env_step": 1935360, "rew": 4110.3599, "rew_std": 657.3085, "Agent": "NPG"}, {"env_step": 1966080, "rew": 3851.8914, "rew_std": 745.2079, "Agent": "NPG"}, {"env_step": 1996800, "rew": 3998.0014, "rew_std": 650.7886, "Agent": "NPG"}, {"env_step": 2027520, "rew": 3739.3657, "rew_std": 715.6727, "Agent": "NPG"}, {"env_step": 2058240, "rew": 4230.5764, "rew_std": 578.5901, "Agent": "NPG"}, {"env_step": 2088960, "rew": 4140.935, "rew_std": 624.3245, "Agent": "NPG"}, {"env_step": 2119680, "rew": 4228.9808, "rew_std": 573.9637, "Agent": "NPG"}, {"env_step": 2150400, "rew": 4142.6649, "rew_std": 419.5781, "Agent": "NPG"}, {"env_step": 2181120, "rew": 4089.8147, "rew_std": 302.9776, "Agent": "NPG"}, {"env_step": 2211840, "rew": 4276.8854, "rew_std": 674.4424, "Agent": "NPG"}, {"env_step": 2242560, "rew": 4312.1654, "rew_std": 567.5481, "Agent": "NPG"}, {"env_step": 2273280, "rew": 4224.3566, "rew_std": 445.0954, "Agent": "NPG"}, {"env_step": 2304000, "rew": 3878.7632, "rew_std": 866.3551, "Agent": "NPG"}, {"env_step": 2334720, "rew": 4001.3801, "rew_std": 478.1705, "Agent": "NPG"}, {"env_step": 2365440, "rew": 4284.5594, "rew_std": 448.8121, "Agent": "NPG"}, {"env_step": 2396160, "rew": 4406.6923, "rew_std": 516.8683, "Agent": "NPG"}, {"env_step": 2426880, "rew": 4145.9032, "rew_std": 519.4268, "Agent": "NPG"}, {"env_step": 2457600, "rew": 4241.7632, "rew_std": 685.6791, "Agent": "NPG"}, {"env_step": 2488320, "rew": 4154.1219, "rew_std": 437.1567, "Agent": "NPG"}, {"env_step": 2519040, "rew": 4495.7374, "rew_std": 449.1806, "Agent": "NPG"}, {"env_step": 2549760, "rew": 4289.7487, "rew_std": 526.7084, "Agent": "NPG"}, {"env_step": 2580480, "rew": 4246.0211, "rew_std": 530.6265, "Agent": "NPG"}, {"env_step": 2611200, "rew": 4463.8985, "rew_std": 572.8797, "Agent": "NPG"}, {"env_step": 2641920, "rew": 4298.6282, "rew_std": 665.9416, "Agent": "NPG"}, {"env_step": 2672640, "rew": 4485.5394, "rew_std": 359.9158, "Agent": "NPG"}, {"env_step": 2703360, "rew": 4820.597, "rew_std": 318.5627, "Agent": "NPG"}, {"env_step": 2734080, "rew": 4540.1727, "rew_std": 412.9545, "Agent": "NPG"}, {"env_step": 2764800, "rew": 4500.1939, "rew_std": 596.1343, "Agent": "NPG"}, {"env_step": 2795520, "rew": 4664.0692, "rew_std": 426.6286, "Agent": "NPG"}, {"env_step": 2826240, "rew": 4516.467, "rew_std": 435.656, "Agent": "NPG"}, {"env_step": 2856960, "rew": 4545.139, "rew_std": 496.411, "Agent": "NPG"}, {"env_step": 2887680, "rew": 4444.0518, "rew_std": 596.6213, "Agent": "NPG"}, {"env_step": 2918400, "rew": 4658.8307, "rew_std": 652.0561, "Agent": "NPG"}, {"env_step": 2949120, "rew": 4509.4439, "rew_std": 466.3483, "Agent": "NPG"}, {"env_step": 2979840, "rew": 4589.8375, "rew_std": 414.0363, "Agent": "NPG"}, {"env_step": 3010560, "rew": 4693.0902, "rew_std": 490.4674, "Agent": "NPG"}, {"env_step": 3041280, "rew": 4678.8152, "rew_std": 313.0054, "Agent": "NPG"}, {"env_step": 3072000, "rew": 4736.6273, "rew_std": 376.7205, "Agent": "NPG"}, {"env_step": 0, "rew": -274.9585, "rew_std": 30.3947, "Agent": "PPO"}, {"env_step": 30720, "rew": 11.9799, "rew_std": 55.2327, "Agent": "PPO"}, {"env_step": 61440, "rew": 345.7298, "rew_std": 139.9961, "Agent": "PPO"}, {"env_step": 92160, "rew": 909.6493, "rew_std": 303.9931, "Agent": "PPO"}, {"env_step": 122880, "rew": 1370.7231, "rew_std": 466.5559, "Agent": "PPO"}, {"env_step": 153600, "rew": 1978.4231, "rew_std": 768.0638, "Agent": "PPO"}, {"env_step": 184320, "rew": 2466.3956, "rew_std": 727.2613, "Agent": "PPO"}, {"env_step": 215040, "rew": 2932.2232, "rew_std": 768.6621, "Agent": "PPO"}, {"env_step": 245760, "rew": 3256.0578, "rew_std": 791.5371, "Agent": "PPO"}, {"env_step": 276480, "rew": 3608.5002, "rew_std": 913.1391, "Agent": "PPO"}, {"env_step": 307200, "rew": 3927.2865, "rew_std": 996.2931, "Agent": "PPO"}, {"env_step": 337920, "rew": 4036.1874, "rew_std": 889.6304, "Agent": "PPO"}, {"env_step": 368640, "rew": 4211.7008, "rew_std": 967.32, "Agent": "PPO"}, {"env_step": 399360, "rew": 4469.8477, "rew_std": 1067.6662, "Agent": "PPO"}, {"env_step": 430080, "rew": 4508.921, "rew_std": 952.455, "Agent": "PPO"}, {"env_step": 460800, "rew": 4718.2123, "rew_std": 1095.8106, "Agent": "PPO"}, {"env_step": 491520, "rew": 4877.9208, "rew_std": 1055.6683, "Agent": "PPO"}, {"env_step": 522240, "rew": 4896.2957, "rew_std": 980.5305, "Agent": "PPO"}, {"env_step": 552960, "rew": 5173.8165, "rew_std": 1106.9123, "Agent": "PPO"}, {"env_step": 583680, "rew": 5164.5684, "rew_std": 1146.3997, "Agent": "PPO"}, {"env_step": 614400, "rew": 5291.8974, "rew_std": 1195.9413, "Agent": "PPO"}, {"env_step": 645120, "rew": 5258.5054, "rew_std": 1149.9965, "Agent": "PPO"}, {"env_step": 675840, "rew": 5389.2988, "rew_std": 1304.1797, "Agent": "PPO"}, {"env_step": 706560, "rew": 5290.879, "rew_std": 1152.4471, "Agent": "PPO"}, {"env_step": 737280, "rew": 5483.6851, "rew_std": 1062.3022, "Agent": "PPO"}, {"env_step": 768000, "rew": 5235.702, "rew_std": 1092.4269, "Agent": "PPO"}, {"env_step": 798720, "rew": 5380.5798, "rew_std": 1149.1554, "Agent": "PPO"}, {"env_step": 829440, "rew": 5609.442, "rew_std": 1139.8926, "Agent": "PPO"}, {"env_step": 860160, "rew": 5535.6169, "rew_std": 1188.8048, "Agent": "PPO"}, {"env_step": 890880, "rew": 5733.831, "rew_std": 1221.6438, "Agent": "PPO"}, {"env_step": 921600, "rew": 5783.8931, "rew_std": 1243.9648, "Agent": "PPO"}, {"env_step": 952320, "rew": 5636.1712, "rew_std": 1398.1925, "Agent": "PPO"}, {"env_step": 983040, "rew": 5780.2176, "rew_std": 1267.7102, "Agent": "PPO"}, {"env_step": 1013760, "rew": 5649.9171, "rew_std": 1226.3424, "Agent": "PPO"}, {"env_step": 1044480, "rew": 5929.9898, "rew_std": 1158.9523, "Agent": "PPO"}, {"env_step": 1075200, "rew": 5961.6938, "rew_std": 1207.4791, "Agent": "PPO"}, {"env_step": 1105920, "rew": 6070.5436, "rew_std": 1286.7158, "Agent": "PPO"}, {"env_step": 1136640, "rew": 6087.6435, "rew_std": 1203.064, "Agent": "PPO"}, {"env_step": 1167360, "rew": 6084.3965, "rew_std": 1086.3397, "Agent": "PPO"}, {"env_step": 1198080, "rew": 6138.0199, "rew_std": 1164.6205, "Agent": "PPO"}, {"env_step": 1228800, "rew": 6103.6806, "rew_std": 1104.3936, "Agent": "PPO"}, {"env_step": 1259520, "rew": 6272.6565, "rew_std": 1283.0591, "Agent": "PPO"}, {"env_step": 1290240, "rew": 5979.2705, "rew_std": 1387.8467, "Agent": "PPO"}, {"env_step": 1320960, "rew": 6270.4063, "rew_std": 1344.6734, "Agent": "PPO"}, {"env_step": 1351680, "rew": 6311.9153, "rew_std": 1318.2905, "Agent": "PPO"}, {"env_step": 1382400, "rew": 6457.731, "rew_std": 1296.4794, "Agent": "PPO"}, {"env_step": 1413120, "rew": 6391.7267, "rew_std": 1265.2202, "Agent": "PPO"}, {"env_step": 1443840, "rew": 6494.0974, "rew_std": 1286.5803, "Agent": "PPO"}, {"env_step": 1474560, "rew": 6443.9407, "rew_std": 1231.9738, "Agent": "PPO"}, {"env_step": 1505280, "rew": 6489.5079, "rew_std": 1387.1682, "Agent": "PPO"}, {"env_step": 1536000, "rew": 6424.721, "rew_std": 1264.9294, "Agent": "PPO"}, {"env_step": 1566720, "rew": 6514.5716, "rew_std": 1267.179, "Agent": "PPO"}, {"env_step": 1597440, "rew": 6442.9185, "rew_std": 1254.8738, "Agent": "PPO"}, {"env_step": 1628160, "rew": 6519.5557, "rew_std": 1200.6957, "Agent": "PPO"}, {"env_step": 1658880, "rew": 6612.5312, "rew_std": 1318.4577, "Agent": "PPO"}, {"env_step": 1689600, "rew": 6621.729, "rew_std": 1408.4337, "Agent": "PPO"}, {"env_step": 1720320, "rew": 6733.1272, "rew_std": 1352.7413, "Agent": "PPO"}, {"env_step": 1751040, "rew": 6598.918, "rew_std": 1249.3865, "Agent": "PPO"}, {"env_step": 1781760, "rew": 6690.3299, "rew_std": 1359.7003, "Agent": "PPO"}, {"env_step": 1812480, "rew": 6603.2403, "rew_std": 1293.2844, "Agent": "PPO"}, {"env_step": 1843200, "rew": 6756.5714, "rew_std": 1222.5844, "Agent": "PPO"}, {"env_step": 1873920, "rew": 6652.8604, "rew_std": 1413.5678, "Agent": "PPO"}, {"env_step": 1904640, "rew": 6703.5758, "rew_std": 1361.8639, "Agent": "PPO"}, {"env_step": 1935360, "rew": 6796.892, "rew_std": 1308.253, "Agent": "PPO"}, {"env_step": 1966080, "rew": 6618.2339, "rew_std": 1212.2791, "Agent": "PPO"}, {"env_step": 1996800, "rew": 6760.9166, "rew_std": 1445.6126, "Agent": "PPO"}, {"env_step": 2027520, "rew": 6657.0263, "rew_std": 1416.5504, "Agent": "PPO"}, {"env_step": 2058240, "rew": 6797.0142, "rew_std": 1369.9286, "Agent": "PPO"}, {"env_step": 2088960, "rew": 6941.3041, "rew_std": 1424.6141, "Agent": "PPO"}, {"env_step": 2119680, "rew": 6679.1551, "rew_std": 1502.0568, "Agent": "PPO"}, {"env_step": 2150400, "rew": 6877.6658, "rew_std": 1419.0989, "Agent": "PPO"}, {"env_step": 2181120, "rew": 6893.2083, "rew_std": 1362.4631, "Agent": "PPO"}, {"env_step": 2211840, "rew": 6884.2739, "rew_std": 1537.1663, "Agent": "PPO"}, {"env_step": 2242560, "rew": 6544.0137, "rew_std": 1765.0257, "Agent": "PPO"}, {"env_step": 2273280, "rew": 6873.6828, "rew_std": 1544.1919, "Agent": "PPO"}, {"env_step": 2304000, "rew": 6930.433, "rew_std": 1425.9817, "Agent": "PPO"}, {"env_step": 2334720, "rew": 6970.6427, "rew_std": 1612.793, "Agent": "PPO"}, {"env_step": 2365440, "rew": 6976.7841, "rew_std": 1502.4346, "Agent": "PPO"}, {"env_step": 2396160, "rew": 6956.7531, "rew_std": 1521.8277, "Agent": "PPO"}, {"env_step": 2426880, "rew": 6923.9605, "rew_std": 1502.0078, "Agent": "PPO"}, {"env_step": 2457600, "rew": 7076.802, "rew_std": 1461.5333, "Agent": "PPO"}, {"env_step": 2488320, "rew": 6910.2399, "rew_std": 1554.5215, "Agent": "PPO"}, {"env_step": 2519040, "rew": 7015.2602, "rew_std": 1620.3088, "Agent": "PPO"}, {"env_step": 2549760, "rew": 7044.4279, "rew_std": 1445.5683, "Agent": "PPO"}, {"env_step": 2580480, "rew": 7149.1791, "rew_std": 1366.216, "Agent": "PPO"}, {"env_step": 2611200, "rew": 7272.657, "rew_std": 1544.8839, "Agent": "PPO"}, {"env_step": 2641920, "rew": 7315.5085, "rew_std": 1382.7119, "Agent": "PPO"}, {"env_step": 2672640, "rew": 7118.8575, "rew_std": 1598.8056, "Agent": "PPO"}, {"env_step": 2703360, "rew": 7207.8444, "rew_std": 1569.6635, "Agent": "PPO"}, {"env_step": 2734080, "rew": 7174.3004, "rew_std": 1515.153, "Agent": "PPO"}, {"env_step": 2764800, "rew": 7262.0208, "rew_std": 1514.371, "Agent": "PPO"}, {"env_step": 2795520, "rew": 7172.3527, "rew_std": 1623.1998, "Agent": "PPO"}, {"env_step": 2826240, "rew": 7233.2708, "rew_std": 1585.3007, "Agent": "PPO"}, {"env_step": 2856960, "rew": 7217.521, "rew_std": 1510.6647, "Agent": "PPO"}, {"env_step": 2887680, "rew": 7293.4964, "rew_std": 1417.3943, "Agent": "PPO"}, {"env_step": 2918400, "rew": 7337.4447, "rew_std": 1508.2475, "Agent": "PPO"}, {"env_step": 2949120, "rew": 7153.7914, "rew_std": 1587.5787, "Agent": "PPO"}, {"env_step": 2979840, "rew": 7241.567, "rew_std": 1467.1855, "Agent": "PPO"}, {"env_step": 3010560, "rew": 7274.2529, "rew_std": 1480.4173, "Agent": "PPO"}, {"env_step": 3041280, "rew": 7236.2394, "rew_std": 1590.6492, "Agent": "PPO"}, {"env_step": 3072000, "rew": 7194.2608, "rew_std": 1404.8626, "Agent": "PPO"}, {"env_step": 0, "rew": -239.2134, "rew_std": 26.5314, "Agent": "REINFORCE"}, {"env_step": 30720, "rew": -130.9784, "rew_std": 17.5175, "Agent": "REINFORCE"}, {"env_step": 61440, "rew": -48.2173, "rew_std": 19.3685, "Agent": "REINFORCE"}, {"env_step": 92160, "rew": -52.4932, "rew_std": 35.6246, "Agent": "REINFORCE"}, {"env_step": 122880, "rew": -65.4243, "rew_std": 29.0437, "Agent": "REINFORCE"}, {"env_step": 153600, "rew": -72.0306, "rew_std": 36.0845, "Agent": "REINFORCE"}, {"env_step": 184320, "rew": 0.4784, "rew_std": 67.6205, "Agent": "REINFORCE"}, {"env_step": 215040, "rew": 13.6712, "rew_std": 101.5885, "Agent": "REINFORCE"}, {"env_step": 245760, "rew": 139.6784, "rew_std": 121.8181, "Agent": "REINFORCE"}, {"env_step": 276480, "rew": 130.5103, "rew_std": 191.4711, "Agent": "REINFORCE"}, {"env_step": 307200, "rew": 114.7512, "rew_std": 190.9939, "Agent": "REINFORCE"}, {"env_step": 337920, "rew": 68.9615, "rew_std": 121.306, "Agent": "REINFORCE"}, {"env_step": 368640, "rew": 109.9668, "rew_std": 138.3891, "Agent": "REINFORCE"}, {"env_step": 399360, "rew": 126.0045, "rew_std": 178.0439, "Agent": "REINFORCE"}, {"env_step": 430080, "rew": 204.6715, "rew_std": 161.6187, "Agent": "REINFORCE"}, {"env_step": 460800, "rew": 272.3794, "rew_std": 130.6516, "Agent": "REINFORCE"}, {"env_step": 491520, "rew": 363.4219, "rew_std": 141.1197, "Agent": "REINFORCE"}, {"env_step": 522240, "rew": 323.1122, "rew_std": 199.7319, "Agent": "REINFORCE"}, {"env_step": 552960, "rew": 336.5633, "rew_std": 202.8921, "Agent": "REINFORCE"}, {"env_step": 583680, "rew": 350.5812, "rew_std": 196.8854, "Agent": "REINFORCE"}, {"env_step": 614400, "rew": 371.6283, "rew_std": 190.2861, "Agent": "REINFORCE"}, {"env_step": 645120, "rew": 403.8344, "rew_std": 176.4811, "Agent": "REINFORCE"}, {"env_step": 675840, "rew": 429.8125, "rew_std": 182.3272, "Agent": "REINFORCE"}, {"env_step": 706560, "rew": 445.647, "rew_std": 156.3292, "Agent": "REINFORCE"}, {"env_step": 737280, "rew": 409.2769, "rew_std": 187.4834, "Agent": "REINFORCE"}, {"env_step": 768000, "rew": 399.6021, "rew_std": 222.1071, "Agent": "REINFORCE"}, {"env_step": 798720, "rew": 402.4893, "rew_std": 243.7293, "Agent": "REINFORCE"}, {"env_step": 829440, "rew": 436.1274, "rew_std": 245.7288, "Agent": "REINFORCE"}, {"env_step": 860160, "rew": 434.0936, "rew_std": 248.5055, "Agent": "REINFORCE"}, {"env_step": 890880, "rew": 422.6199, "rew_std": 266.891, "Agent": "REINFORCE"}, {"env_step": 921600, "rew": 458.7628, "rew_std": 234.5876, "Agent": "REINFORCE"}, {"env_step": 952320, "rew": 481.243, "rew_std": 243.8428, "Agent": "REINFORCE"}, {"env_step": 983040, "rew": 512.3078, "rew_std": 209.158, "Agent": "REINFORCE"}, {"env_step": 1013760, "rew": 525.5006, "rew_std": 181.8258, "Agent": "REINFORCE"}, {"env_step": 1044480, "rew": 532.0321, "rew_std": 198.5119, "Agent": "REINFORCE"}, {"env_step": 1075200, "rew": 541.2009, "rew_std": 191.9939, "Agent": "REINFORCE"}, {"env_step": 1105920, "rew": 552.9654, "rew_std": 191.0248, "Agent": "REINFORCE"}, {"env_step": 1136640, "rew": 549.1181, "rew_std": 197.3171, "Agent": "REINFORCE"}, {"env_step": 1167360, "rew": 604.0583, "rew_std": 174.7335, "Agent": "REINFORCE"}, {"env_step": 1198080, "rew": 594.4701, "rew_std": 154.8017, "Agent": "REINFORCE"}, {"env_step": 1228800, "rew": 606.9413, "rew_std": 156.8321, "Agent": "REINFORCE"}, {"env_step": 1259520, "rew": 620.9988, "rew_std": 143.3749, "Agent": "REINFORCE"}, {"env_step": 1290240, "rew": 634.8226, "rew_std": 140.0081, "Agent": "REINFORCE"}, {"env_step": 1320960, "rew": 635.5886, "rew_std": 129.6849, "Agent": "REINFORCE"}, {"env_step": 1351680, "rew": 641.4025, "rew_std": 119.5029, "Agent": "REINFORCE"}, {"env_step": 1382400, "rew": 631.3879, "rew_std": 128.1176, "Agent": "REINFORCE"}, {"env_step": 1413120, "rew": 657.6043, "rew_std": 124.8992, "Agent": "REINFORCE"}, {"env_step": 1443840, "rew": 675.7489, "rew_std": 119.027, "Agent": "REINFORCE"}, {"env_step": 1474560, "rew": 662.4531, "rew_std": 105.9602, "Agent": "REINFORCE"}, {"env_step": 1505280, "rew": 666.6031, "rew_std": 103.0068, "Agent": "REINFORCE"}, {"env_step": 1536000, "rew": 683.1362, "rew_std": 107.026, "Agent": "REINFORCE"}, {"env_step": 1566720, "rew": 686.2614, "rew_std": 112.2725, "Agent": "REINFORCE"}, {"env_step": 1597440, "rew": 696.8179, "rew_std": 98.897, "Agent": "REINFORCE"}, {"env_step": 1628160, "rew": 702.4769, "rew_std": 94.8873, "Agent": "REINFORCE"}, {"env_step": 1658880, "rew": 704.4676, "rew_std": 87.216, "Agent": "REINFORCE"}, {"env_step": 1689600, "rew": 733.5609, "rew_std": 85.4199, "Agent": "REINFORCE"}, {"env_step": 1720320, "rew": 739.9737, "rew_std": 74.0321, "Agent": "REINFORCE"}, {"env_step": 1751040, "rew": 727.4521, "rew_std": 62.8634, "Agent": "REINFORCE"}, {"env_step": 1781760, "rew": 732.5071, "rew_std": 71.188, "Agent": "REINFORCE"}, {"env_step": 1812480, "rew": 748.7167, "rew_std": 76.5575, "Agent": "REINFORCE"}, {"env_step": 1843200, "rew": 737.4527, "rew_std": 72.1915, "Agent": "REINFORCE"}, {"env_step": 1873920, "rew": 754.2162, "rew_std": 73.0255, "Agent": "REINFORCE"}, {"env_step": 1904640, "rew": 756.587, "rew_std": 80.629, "Agent": "REINFORCE"}, {"env_step": 1935360, "rew": 725.8417, "rew_std": 120.566, "Agent": "REINFORCE"}, {"env_step": 1966080, "rew": 769.9105, "rew_std": 63.3313, "Agent": "REINFORCE"}, {"env_step": 1996800, "rew": 755.0456, "rew_std": 47.4848, "Agent": "REINFORCE"}, {"env_step": 2027520, "rew": 780.3756, "rew_std": 56.5651, "Agent": "REINFORCE"}, {"env_step": 2058240, "rew": 769.5733, "rew_std": 67.2945, "Agent": "REINFORCE"}, {"env_step": 2088960, "rew": 781.3064, "rew_std": 72.9166, "Agent": "REINFORCE"}, {"env_step": 2119680, "rew": 781.6407, "rew_std": 47.8687, "Agent": "REINFORCE"}, {"env_step": 2150400, "rew": 798.3952, "rew_std": 72.5853, "Agent": "REINFORCE"}, {"env_step": 2181120, "rew": 793.7055, "rew_std": 57.7114, "Agent": "REINFORCE"}, {"env_step": 2211840, "rew": 788.6205, "rew_std": 65.4386, "Agent": "REINFORCE"}, {"env_step": 2242560, "rew": 803.8613, "rew_std": 46.9137, "Agent": "REINFORCE"}, {"env_step": 2273280, "rew": 782.8062, "rew_std": 58.0125, "Agent": "REINFORCE"}, {"env_step": 2304000, "rew": 795.8051, "rew_std": 55.088, "Agent": "REINFORCE"}, {"env_step": 2334720, "rew": 809.0786, "rew_std": 41.5371, "Agent": "REINFORCE"}, {"env_step": 2365440, "rew": 812.1326, "rew_std": 50.8964, "Agent": "REINFORCE"}, {"env_step": 2396160, "rew": 813.0989, "rew_std": 60.4955, "Agent": "REINFORCE"}, {"env_step": 2426880, "rew": 810.6179, "rew_std": 52.8278, "Agent": "REINFORCE"}, {"env_step": 2457600, "rew": 809.6877, "rew_std": 54.128, "Agent": "REINFORCE"}, {"env_step": 2488320, "rew": 831.7716, "rew_std": 46.8229, "Agent": "REINFORCE"}, {"env_step": 2519040, "rew": 831.0148, "rew_std": 49.9816, "Agent": "REINFORCE"}, {"env_step": 2549760, "rew": 819.638, "rew_std": 65.2786, "Agent": "REINFORCE"}, {"env_step": 2580480, "rew": 835.1732, "rew_std": 58.8964, "Agent": "REINFORCE"}, {"env_step": 2611200, "rew": 849.7141, "rew_std": 45.4597, "Agent": "REINFORCE"}, {"env_step": 2641920, "rew": 858.2384, "rew_std": 49.3522, "Agent": "REINFORCE"}, {"env_step": 2672640, "rew": 862.0843, "rew_std": 47.91, "Agent": "REINFORCE"}, {"env_step": 2703360, "rew": 855.118, "rew_std": 31.717, "Agent": "REINFORCE"}, {"env_step": 2734080, "rew": 865.496, "rew_std": 51.6423, "Agent": "REINFORCE"}, {"env_step": 2764800, "rew": 852.2358, "rew_std": 49.6403, "Agent": "REINFORCE"}, {"env_step": 2795520, "rew": 846.0969, "rew_std": 29.3903, "Agent": "REINFORCE"}, {"env_step": 2826240, "rew": 843.4356, "rew_std": 35.9265, "Agent": "REINFORCE"}, {"env_step": 2856960, "rew": 876.8606, "rew_std": 36.6545, "Agent": "REINFORCE"}, {"env_step": 2887680, "rew": 878.6943, "rew_std": 35.3836, "Agent": "REINFORCE"}, {"env_step": 2918400, "rew": 883.9985, "rew_std": 40.9765, "Agent": "REINFORCE"}, {"env_step": 2949120, "rew": 879.088, "rew_std": 36.9194, "Agent": "REINFORCE"}, {"env_step": 2979840, "rew": 883.1086, "rew_std": 51.5751, "Agent": "REINFORCE"}, {"env_step": 3010560, "rew": 878.5629, "rew_std": 51.4401, "Agent": "REINFORCE"}, {"env_step": 3041280, "rew": 876.7223, "rew_std": 54.3786, "Agent": "REINFORCE"}, {"env_step": 3072000, "rew": 881.5336, "rew_std": 83.5794, "Agent": "REINFORCE"}, {"env_step": 3102720, "rew": 867.0418, "rew_std": 82.998, "Agent": "REINFORCE"}, {"env_step": 3133440, "rew": 877.304, "rew_std": 75.1667, "Agent": "REINFORCE"}, {"env_step": 3164160, "rew": 876.2122, "rew_std": 93.2066, "Agent": "REINFORCE"}, {"env_step": 3194880, "rew": 822.7774, "rew_std": 179.4039, "Agent": "REINFORCE"}, {"env_step": 3225600, "rew": 865.225, "rew_std": 95.6548, "Agent": "REINFORCE"}, {"env_step": 3256320, "rew": 876.3886, "rew_std": 65.7255, "Agent": "REINFORCE"}, {"env_step": 3287040, "rew": 901.4318, "rew_std": 44.2542, "Agent": "REINFORCE"}, {"env_step": 3317760, "rew": 906.5281, "rew_std": 40.9878, "Agent": "REINFORCE"}, {"env_step": 3348480, "rew": 902.8887, "rew_std": 42.6048, "Agent": "REINFORCE"}, {"env_step": 3379200, "rew": 887.7884, "rew_std": 55.2236, "Agent": "REINFORCE"}, {"env_step": 3409920, "rew": 890.9958, "rew_std": 45.3819, "Agent": "REINFORCE"}, {"env_step": 3440640, "rew": 905.9487, "rew_std": 59.0244, "Agent": "REINFORCE"}, {"env_step": 3471360, "rew": 878.4631, "rew_std": 77.4708, "Agent": "REINFORCE"}, {"env_step": 3502080, "rew": 918.9451, "rew_std": 68.617, "Agent": "REINFORCE"}, {"env_step": 3532800, "rew": 912.5092, "rew_std": 56.8806, "Agent": "REINFORCE"}, {"env_step": 3563520, "rew": 930.3603, "rew_std": 47.639, "Agent": "REINFORCE"}, {"env_step": 3594240, "rew": 937.8539, "rew_std": 46.8265, "Agent": "REINFORCE"}, {"env_step": 3624960, "rew": 933.9781, "rew_std": 54.7217, "Agent": "REINFORCE"}, {"env_step": 3655680, "rew": 924.4348, "rew_std": 46.9608, "Agent": "REINFORCE"}, {"env_step": 3686400, "rew": 932.5993, "rew_std": 61.1287, "Agent": "REINFORCE"}, {"env_step": 3717120, "rew": 930.562, "rew_std": 39.1545, "Agent": "REINFORCE"}, {"env_step": 3747840, "rew": 939.6507, "rew_std": 50.0547, "Agent": "REINFORCE"}, {"env_step": 3778560, "rew": 931.6101, "rew_std": 64.34, "Agent": "REINFORCE"}, {"env_step": 3809280, "rew": 893.6656, "rew_std": 91.2603, "Agent": "REINFORCE"}, {"env_step": 3840000, "rew": 933.943, "rew_std": 51.9606, "Agent": "REINFORCE"}, {"env_step": 3870720, "rew": 937.8593, "rew_std": 58.272, "Agent": "REINFORCE"}, {"env_step": 3901440, "rew": 923.1785, "rew_std": 46.8948, "Agent": "REINFORCE"}, {"env_step": 3932160, "rew": 919.5106, "rew_std": 79.6043, "Agent": "REINFORCE"}, {"env_step": 3962880, "rew": 937.6189, "rew_std": 35.4825, "Agent": "REINFORCE"}, {"env_step": 3993600, "rew": 941.5022, "rew_std": 39.9812, "Agent": "REINFORCE"}, {"env_step": 4024320, "rew": 927.7056, "rew_std": 43.8473, "Agent": "REINFORCE"}, {"env_step": 4055040, "rew": 955.5857, "rew_std": 40.0532, "Agent": "REINFORCE"}, {"env_step": 4085760, "rew": 954.8072, "rew_std": 47.7404, "Agent": "REINFORCE"}, {"env_step": 4116480, "rew": 968.9528, "rew_std": 47.641, "Agent": "REINFORCE"}, {"env_step": 4147200, "rew": 927.58, "rew_std": 60.9435, "Agent": "REINFORCE"}, {"env_step": 4177920, "rew": 951.3564, "rew_std": 35.3196, "Agent": "REINFORCE"}, {"env_step": 4208640, "rew": 949.8772, "rew_std": 44.6866, "Agent": "REINFORCE"}, {"env_step": 4239360, "rew": 923.3382, "rew_std": 59.9723, "Agent": "REINFORCE"}, {"env_step": 4270080, "rew": 947.1497, "rew_std": 46.7235, "Agent": "REINFORCE"}, {"env_step": 4300800, "rew": 930.2156, "rew_std": 96.1385, "Agent": "REINFORCE"}, {"env_step": 4331520, "rew": 947.0682, "rew_std": 44.4635, "Agent": "REINFORCE"}, {"env_step": 4362240, "rew": 913.4538, "rew_std": 106.4576, "Agent": "REINFORCE"}, {"env_step": 4392960, "rew": 956.4173, "rew_std": 72.454, "Agent": "REINFORCE"}, {"env_step": 4423680, "rew": 986.112, "rew_std": 53.2538, "Agent": "REINFORCE"}, {"env_step": 4454400, "rew": 980.6774, "rew_std": 55.4501, "Agent": "REINFORCE"}, {"env_step": 4485120, "rew": 969.6189, "rew_std": 45.7465, "Agent": "REINFORCE"}, {"env_step": 4515840, "rew": 961.9398, "rew_std": 44.0191, "Agent": "REINFORCE"}, {"env_step": 4546560, "rew": 977.3772, "rew_std": 46.2519, "Agent": "REINFORCE"}, {"env_step": 4577280, "rew": 960.5113, "rew_std": 43.8153, "Agent": "REINFORCE"}, {"env_step": 4608000, "rew": 975.5923, "rew_std": 42.7543, "Agent": "REINFORCE"}, {"env_step": 4638720, "rew": 978.18, "rew_std": 60.7165, "Agent": "REINFORCE"}, {"env_step": 4669440, "rew": 994.968, "rew_std": 52.9976, "Agent": "REINFORCE"}, {"env_step": 4700160, "rew": 992.85, "rew_std": 53.1501, "Agent": "REINFORCE"}, {"env_step": 4730880, "rew": 964.6797, "rew_std": 46.663, "Agent": "REINFORCE"}, {"env_step": 4761600, "rew": 983.9115, "rew_std": 52.2566, "Agent": "REINFORCE"}, {"env_step": 4792320, "rew": 990.3478, "rew_std": 40.05, "Agent": "REINFORCE"}, {"env_step": 4823040, "rew": 997.0132, "rew_std": 41.0583, "Agent": "REINFORCE"}, {"env_step": 4853760, "rew": 993.6661, "rew_std": 42.785, "Agent": "REINFORCE"}, {"env_step": 4884480, "rew": 1011.8642, "rew_std": 27.5827, "Agent": "REINFORCE"}, {"env_step": 4915200, "rew": 1005.24, "rew_std": 22.0347, "Agent": "REINFORCE"}, {"env_step": 4945920, "rew": 995.1774, "rew_std": 38.7399, "Agent": "REINFORCE"}, {"env_step": 4976640, "rew": 993.0942, "rew_std": 45.0712, "Agent": "REINFORCE"}, {"env_step": 5007360, "rew": 1013.1125, "rew_std": 60.5238, "Agent": "REINFORCE"}, {"env_step": 5038080, "rew": 996.6935, "rew_std": 44.7954, "Agent": "REINFORCE"}, {"env_step": 5068800, "rew": 995.8926, "rew_std": 58.3237, "Agent": "REINFORCE"}, {"env_step": 5099520, "rew": 1010.5929, "rew_std": 54.3951, "Agent": "REINFORCE"}, {"env_step": 5130240, "rew": 994.6702, "rew_std": 69.0843, "Agent": "REINFORCE"}, {"env_step": 5160960, "rew": 997.2196, "rew_std": 39.166, "Agent": "REINFORCE"}, {"env_step": 5191680, "rew": 1016.5374, "rew_std": 54.9657, "Agent": "REINFORCE"}, {"env_step": 5222400, "rew": 1017.5374, "rew_std": 62.1809, "Agent": "REINFORCE"}, {"env_step": 5253120, "rew": 1015.6075, "rew_std": 53.5472, "Agent": "REINFORCE"}, {"env_step": 5283840, "rew": 1016.8015, "rew_std": 52.6332, "Agent": "REINFORCE"}, {"env_step": 5314560, "rew": 1002.2621, "rew_std": 57.4378, "Agent": "REINFORCE"}, {"env_step": 5345280, "rew": 1001.0012, "rew_std": 100.386, "Agent": "REINFORCE"}, {"env_step": 5376000, "rew": 1022.3522, "rew_std": 66.3868, "Agent": "REINFORCE"}, {"env_step": 5406720, "rew": 1000.0438, "rew_std": 51.6201, "Agent": "REINFORCE"}, {"env_step": 5437440, "rew": 1011.1049, "rew_std": 62.4932, "Agent": "REINFORCE"}, {"env_step": 5468160, "rew": 1015.1676, "rew_std": 89.5522, "Agent": "REINFORCE"}, {"env_step": 5498880, "rew": 1006.4157, "rew_std": 58.868, "Agent": "REINFORCE"}, {"env_step": 5529600, "rew": 1021.2901, "rew_std": 48.5136, "Agent": "REINFORCE"}, {"env_step": 5560320, "rew": 1034.2604, "rew_std": 51.6751, "Agent": "REINFORCE"}, {"env_step": 5591040, "rew": 1012.779, "rew_std": 60.1441, "Agent": "REINFORCE"}, {"env_step": 5621760, "rew": 1019.7712, "rew_std": 50.3584, "Agent": "REINFORCE"}, {"env_step": 5652480, "rew": 1028.3674, "rew_std": 37.5526, "Agent": "REINFORCE"}, {"env_step": 5683200, "rew": 1022.9339, "rew_std": 42.2359, "Agent": "REINFORCE"}, {"env_step": 5713920, "rew": 1035.5652, "rew_std": 35.8611, "Agent": "REINFORCE"}, {"env_step": 5744640, "rew": 1034.2733, "rew_std": 40.2398, "Agent": "REINFORCE"}, {"env_step": 5775360, "rew": 1045.4148, "rew_std": 49.8297, "Agent": "REINFORCE"}, {"env_step": 5806080, "rew": 1050.8877, "rew_std": 40.4739, "Agent": "REINFORCE"}, {"env_step": 5836800, "rew": 1039.0886, "rew_std": 36.1023, "Agent": "REINFORCE"}, {"env_step": 5867520, "rew": 1036.9562, "rew_std": 70.2645, "Agent": "REINFORCE"}, {"env_step": 5898240, "rew": 1038.1224, "rew_std": 51.4148, "Agent": "REINFORCE"}, {"env_step": 5928960, "rew": 1043.1975, "rew_std": 55.1165, "Agent": "REINFORCE"}, {"env_step": 5959680, "rew": 1034.1112, "rew_std": 65.8918, "Agent": "REINFORCE"}, {"env_step": 5990400, "rew": 1039.1232, "rew_std": 74.2882, "Agent": "REINFORCE"}, {"env_step": 6021120, "rew": 1026.4448, "rew_std": 72.7191, "Agent": "REINFORCE"}, {"env_step": 6051840, "rew": 1042.3366, "rew_std": 61.0111, "Agent": "REINFORCE"}, {"env_step": 6082560, "rew": 1025.8535, "rew_std": 64.4098, "Agent": "REINFORCE"}, {"env_step": 6113280, "rew": 1034.3692, "rew_std": 66.0265, "Agent": "REINFORCE"}, {"env_step": 6144000, "rew": 1035.6472, "rew_std": 59.5942, "Agent": "REINFORCE"}, {"env_step": 6174720, "rew": 1024.2567, "rew_std": 52.5294, "Agent": "REINFORCE"}, {"env_step": 6205440, "rew": 1035.5613, "rew_std": 51.0109, "Agent": "REINFORCE"}, {"env_step": 6236160, "rew": 1050.8728, "rew_std": 54.2495, "Agent": "REINFORCE"}, {"env_step": 6266880, "rew": 1027.9969, "rew_std": 53.9087, "Agent": "REINFORCE"}, {"env_step": 6297600, "rew": 1058.0868, "rew_std": 43.1176, "Agent": "REINFORCE"}, {"env_step": 6328320, "rew": 1049.4824, "rew_std": 47.3149, "Agent": "REINFORCE"}, {"env_step": 6359040, "rew": 1026.2656, "rew_std": 58.7652, "Agent": "REINFORCE"}, {"env_step": 6389760, "rew": 1063.611, "rew_std": 62.7269, "Agent": "REINFORCE"}, {"env_step": 6420480, "rew": 1013.5708, "rew_std": 87.5254, "Agent": "REINFORCE"}, {"env_step": 6451200, "rew": 1059.3438, "rew_std": 46.8292, "Agent": "REINFORCE"}, {"env_step": 6481920, "rew": 1021.049, "rew_std": 81.6385, "Agent": "REINFORCE"}, {"env_step": 6512640, "rew": 1053.7262, "rew_std": 74.1989, "Agent": "REINFORCE"}, {"env_step": 6543360, "rew": 1053.0535, "rew_std": 65.418, "Agent": "REINFORCE"}, {"env_step": 6574080, "rew": 1059.707, "rew_std": 69.9458, "Agent": "REINFORCE"}, {"env_step": 6604800, "rew": 1045.8224, "rew_std": 54.9453, "Agent": "REINFORCE"}, {"env_step": 6635520, "rew": 1047.9718, "rew_std": 59.0243, "Agent": "REINFORCE"}, {"env_step": 6666240, "rew": 1053.1456, "rew_std": 60.2588, "Agent": "REINFORCE"}, {"env_step": 6696960, "rew": 1050.9961, "rew_std": 72.2672, "Agent": "REINFORCE"}, {"env_step": 6727680, "rew": 1055.0399, "rew_std": 73.4961, "Agent": "REINFORCE"}, {"env_step": 6758400, "rew": 1052.8888, "rew_std": 70.5037, "Agent": "REINFORCE"}, {"env_step": 6789120, "rew": 1029.896, "rew_std": 92.459, "Agent": "REINFORCE"}, {"env_step": 6819840, "rew": 1058.1352, "rew_std": 58.4981, "Agent": "REINFORCE"}, {"env_step": 6850560, "rew": 1040.137, "rew_std": 86.8626, "Agent": "REINFORCE"}, {"env_step": 6881280, "rew": 1048.5212, "rew_std": 58.2427, "Agent": "REINFORCE"}, {"env_step": 6912000, "rew": 1056.9189, "rew_std": 56.9239, "Agent": "REINFORCE"}, {"env_step": 6942720, "rew": 1066.5289, "rew_std": 56.7241, "Agent": "REINFORCE"}, {"env_step": 6973440, "rew": 1039.9768, "rew_std": 69.6099, "Agent": "REINFORCE"}, {"env_step": 7004160, "rew": 1055.0471, "rew_std": 48.4431, "Agent": "REINFORCE"}, {"env_step": 7034880, "rew": 1073.3547, "rew_std": 54.2786, "Agent": "REINFORCE"}, {"env_step": 7065600, "rew": 1078.4087, "rew_std": 56.8925, "Agent": "REINFORCE"}, {"env_step": 7096320, "rew": 1062.771, "rew_std": 56.5096, "Agent": "REINFORCE"}, {"env_step": 7127040, "rew": 1064.348, "rew_std": 46.8655, "Agent": "REINFORCE"}, {"env_step": 7157760, "rew": 1069.0247, "rew_std": 53.9928, "Agent": "REINFORCE"}, {"env_step": 7188480, "rew": 1095.3149, "rew_std": 56.0251, "Agent": "REINFORCE"}, {"env_step": 7219200, "rew": 1059.7795, "rew_std": 89.3297, "Agent": "REINFORCE"}, {"env_step": 7249920, "rew": 1075.7866, "rew_std": 48.5306, "Agent": "REINFORCE"}, {"env_step": 7280640, "rew": 1070.4785, "rew_std": 57.9126, "Agent": "REINFORCE"}, {"env_step": 7311360, "rew": 1067.7449, "rew_std": 45.6498, "Agent": "REINFORCE"}, {"env_step": 7342080, "rew": 1068.3339, "rew_std": 51.5933, "Agent": "REINFORCE"}, {"env_step": 7372800, "rew": 1073.2552, "rew_std": 45.2915, "Agent": "REINFORCE"}, {"env_step": 7403520, "rew": 1086.6587, "rew_std": 51.2162, "Agent": "REINFORCE"}, {"env_step": 7434240, "rew": 1078.313, "rew_std": 39.8316, "Agent": "REINFORCE"}, {"env_step": 7464960, "rew": 1060.5985, "rew_std": 44.682, "Agent": "REINFORCE"}, {"env_step": 7495680, "rew": 1075.5627, "rew_std": 50.3273, "Agent": "REINFORCE"}, {"env_step": 7526400, "rew": 1072.449, "rew_std": 63.2418, "Agent": "REINFORCE"}, {"env_step": 7557120, "rew": 1085.9426, "rew_std": 33.5946, "Agent": "REINFORCE"}, {"env_step": 7587840, "rew": 1092.3983, "rew_std": 53.6555, "Agent": "REINFORCE"}, {"env_step": 7618560, "rew": 1078.9839, "rew_std": 54.3774, "Agent": "REINFORCE"}, {"env_step": 7649280, "rew": 1091.3667, "rew_std": 46.3437, "Agent": "REINFORCE"}, {"env_step": 7680000, "rew": 1082.5875, "rew_std": 42.6033, "Agent": "REINFORCE"}, {"env_step": 7710720, "rew": 1066.3451, "rew_std": 61.7616, "Agent": "REINFORCE"}, {"env_step": 7741440, "rew": 1073.6454, "rew_std": 59.707, "Agent": "REINFORCE"}, {"env_step": 7772160, "rew": 1063.3744, "rew_std": 71.0224, "Agent": "REINFORCE"}, {"env_step": 7802880, "rew": 1090.047, "rew_std": 40.6139, "Agent": "REINFORCE"}, {"env_step": 7833600, "rew": 1080.7565, "rew_std": 44.8726, "Agent": "REINFORCE"}, {"env_step": 7864320, "rew": 1065.6765, "rew_std": 46.4016, "Agent": "REINFORCE"}, {"env_step": 7895040, "rew": 1067.6488, "rew_std": 49.9414, "Agent": "REINFORCE"}, {"env_step": 7925760, "rew": 1085.3441, "rew_std": 55.2402, "Agent": "REINFORCE"}, {"env_step": 7956480, "rew": 1062.0563, "rew_std": 49.4121, "Agent": "REINFORCE"}, {"env_step": 7987200, "rew": 1086.9089, "rew_std": 50.1225, "Agent": "REINFORCE"}, {"env_step": 8017920, "rew": 1085.0331, "rew_std": 42.0211, "Agent": "REINFORCE"}, {"env_step": 8048640, "rew": 1076.8434, "rew_std": 43.465, "Agent": "REINFORCE"}, {"env_step": 8079360, "rew": 1097.6033, "rew_std": 44.2847, "Agent": "REINFORCE"}, {"env_step": 8110080, "rew": 1105.4302, "rew_std": 43.7604, "Agent": "REINFORCE"}, {"env_step": 8140800, "rew": 1088.2506, "rew_std": 37.5434, "Agent": "REINFORCE"}, {"env_step": 8171520, "rew": 1080.3139, "rew_std": 45.7188, "Agent": "REINFORCE"}, {"env_step": 8202240, "rew": 1084.8442, "rew_std": 47.4441, "Agent": "REINFORCE"}, {"env_step": 8232960, "rew": 1077.7378, "rew_std": 59.542, "Agent": "REINFORCE"}, {"env_step": 8263680, "rew": 1066.8771, "rew_std": 72.9645, "Agent": "REINFORCE"}, {"env_step": 8294400, "rew": 1096.348, "rew_std": 52.8151, "Agent": "REINFORCE"}, {"env_step": 8325120, "rew": 1071.0871, "rew_std": 50.5231, "Agent": "REINFORCE"}, {"env_step": 8355840, "rew": 1091.951, "rew_std": 44.6643, "Agent": "REINFORCE"}, {"env_step": 8386560, "rew": 1086.8091, "rew_std": 46.2969, "Agent": "REINFORCE"}, {"env_step": 8417280, "rew": 1095.6318, "rew_std": 51.617, "Agent": "REINFORCE"}, {"env_step": 8448000, "rew": 1064.6144, "rew_std": 70.4566, "Agent": "REINFORCE"}, {"env_step": 8478720, "rew": 1097.0543, "rew_std": 45.3195, "Agent": "REINFORCE"}, {"env_step": 8509440, "rew": 1104.6612, "rew_std": 42.8634, "Agent": "REINFORCE"}, {"env_step": 8540160, "rew": 1098.8933, "rew_std": 38.3864, "Agent": "REINFORCE"}, {"env_step": 8570880, "rew": 1086.6912, "rew_std": 50.3266, "Agent": "REINFORCE"}, {"env_step": 8601600, "rew": 1102.2135, "rew_std": 70.933, "Agent": "REINFORCE"}, {"env_step": 8632320, "rew": 1090.1486, "rew_std": 81.3128, "Agent": "REINFORCE"}, {"env_step": 8663040, "rew": 1087.0612, "rew_std": 57.0418, "Agent": "REINFORCE"}, {"env_step": 8693760, "rew": 1107.9893, "rew_std": 51.2113, "Agent": "REINFORCE"}, {"env_step": 8724480, "rew": 1116.9207, "rew_std": 50.9583, "Agent": "REINFORCE"}, {"env_step": 8755200, "rew": 1112.6254, "rew_std": 43.8869, "Agent": "REINFORCE"}, {"env_step": 8785920, "rew": 1118.0138, "rew_std": 37.5083, "Agent": "REINFORCE"}, {"env_step": 8816640, "rew": 1092.8679, "rew_std": 52.3159, "Agent": "REINFORCE"}, {"env_step": 8847360, "rew": 1112.43, "rew_std": 43.5469, "Agent": "REINFORCE"}, {"env_step": 8878080, "rew": 1109.6117, "rew_std": 53.1086, "Agent": "REINFORCE"}, {"env_step": 8908800, "rew": 1091.1253, "rew_std": 55.2791, "Agent": "REINFORCE"}, {"env_step": 8939520, "rew": 1103.3368, "rew_std": 47.0241, "Agent": "REINFORCE"}, {"env_step": 8970240, "rew": 1117.9517, "rew_std": 45.6591, "Agent": "REINFORCE"}, {"env_step": 9000960, "rew": 1106.9619, "rew_std": 41.4317, "Agent": "REINFORCE"}, {"env_step": 9031680, "rew": 1115.1068, "rew_std": 55.2598, "Agent": "REINFORCE"}, {"env_step": 9062400, "rew": 1122.0894, "rew_std": 35.4838, "Agent": "REINFORCE"}, {"env_step": 9093120, "rew": 1127.6883, "rew_std": 46.4658, "Agent": "REINFORCE"}, {"env_step": 9123840, "rew": 1112.8166, "rew_std": 41.8131, "Agent": "REINFORCE"}, {"env_step": 9154560, "rew": 1115.7368, "rew_std": 53.6233, "Agent": "REINFORCE"}, {"env_step": 9185280, "rew": 1095.7732, "rew_std": 39.6533, "Agent": "REINFORCE"}, {"env_step": 9216000, "rew": 1120.5861, "rew_std": 41.3632, "Agent": "REINFORCE"}, {"env_step": 9246720, "rew": 1082.4576, "rew_std": 92.7449, "Agent": "REINFORCE"}, {"env_step": 9277440, "rew": 1129.3393, "rew_std": 45.1588, "Agent": "REINFORCE"}, {"env_step": 9308160, "rew": 1107.1046, "rew_std": 57.317, "Agent": "REINFORCE"}, {"env_step": 9338880, "rew": 1115.6161, "rew_std": 54.6329, "Agent": "REINFORCE"}, {"env_step": 9369600, "rew": 1123.2624, "rew_std": 32.816, "Agent": "REINFORCE"}, {"env_step": 9400320, "rew": 1127.7662, "rew_std": 38.7112, "Agent": "REINFORCE"}, {"env_step": 9431040, "rew": 1122.9372, "rew_std": 35.7091, "Agent": "REINFORCE"}, {"env_step": 9461760, "rew": 1115.9145, "rew_std": 40.5152, "Agent": "REINFORCE"}, {"env_step": 9492480, "rew": 1113.5322, "rew_std": 40.1525, "Agent": "REINFORCE"}, {"env_step": 9523200, "rew": 1136.2791, "rew_std": 35.7975, "Agent": "REINFORCE"}, {"env_step": 9553920, "rew": 1113.7384, "rew_std": 35.8931, "Agent": "REINFORCE"}, {"env_step": 9584640, "rew": 1121.8195, "rew_std": 42.9915, "Agent": "REINFORCE"}, {"env_step": 9615360, "rew": 1130.7352, "rew_std": 35.1201, "Agent": "REINFORCE"}, {"env_step": 9646080, "rew": 1125.1981, "rew_std": 40.9015, "Agent": "REINFORCE"}, {"env_step": 9676800, "rew": 1118.2906, "rew_std": 48.4825, "Agent": "REINFORCE"}, {"env_step": 9707520, "rew": 1089.7356, "rew_std": 58.9498, "Agent": "REINFORCE"}, {"env_step": 9738240, "rew": 1115.0204, "rew_std": 53.2705, "Agent": "REINFORCE"}, {"env_step": 9768960, "rew": 1121.7228, "rew_std": 48.332, "Agent": "REINFORCE"}, {"env_step": 9799680, "rew": 1129.367, "rew_std": 35.2903, "Agent": "REINFORCE"}, {"env_step": 9830400, "rew": 1118.8765, "rew_std": 36.8765, "Agent": "REINFORCE"}, {"env_step": 9861120, "rew": 1113.9437, "rew_std": 69.7432, "Agent": "REINFORCE"}, {"env_step": 9891840, "rew": 1120.8454, "rew_std": 33.3913, "Agent": "REINFORCE"}, {"env_step": 9922560, "rew": 1113.0581, "rew_std": 56.4189, "Agent": "REINFORCE"}, {"env_step": 9953280, "rew": 1120.8607, "rew_std": 33.265, "Agent": "REINFORCE"}, {"env_step": 9984000, "rew": 1107.17, "rew_std": 51.0692, "Agent": "REINFORCE"}, {"env_step": 10014720, "rew": 1123.9591, "rew_std": 39.8303, "Agent": "REINFORCE"}, {"env_step": 10045440, "rew": 1132.0132, "rew_std": 53.0461, "Agent": "REINFORCE"}, {"env_step": 10076160, "rew": 1138.7607, "rew_std": 38.8626, "Agent": "REINFORCE"}, {"env_step": 10106880, "rew": 1125.9321, "rew_std": 35.3656, "Agent": "REINFORCE"}, {"env_step": 10137600, "rew": 1121.423, "rew_std": 45.7195, "Agent": "REINFORCE"}, {"env_step": 10168320, "rew": 1132.5021, "rew_std": 46.9729, "Agent": "REINFORCE"}, {"env_step": 10199040, "rew": 1133.2431, "rew_std": 38.2374, "Agent": "REINFORCE"}, {"env_step": 10229760, "rew": 1125.8592, "rew_std": 40.0745, "Agent": "REINFORCE"}, {"env_step": 0, "rew": -265.1803, "rew_std": 26.8288, "Agent": "A2C"}, {"env_step": 30000, "rew": -151.642, "rew_std": 224.3554, "Agent": "A2C"}, {"env_step": 60000, "rew": -92.9903, "rew_std": 289.8266, "Agent": "A2C"}, {"env_step": 90000, "rew": 38.995, "rew_std": 327.5368, "Agent": "A2C"}, {"env_step": 120000, "rew": 87.1445, "rew_std": 307.6683, "Agent": "A2C"}, {"env_step": 150000, "rew": 220.5334, "rew_std": 379.6231, "Agent": "A2C"}, {"env_step": 180000, "rew": 409.0691, "rew_std": 452.5335, "Agent": "A2C"}, {"env_step": 210000, "rew": 544.2138, "rew_std": 558.6034, "Agent": "A2C"}, {"env_step": 240000, "rew": 514.9554, "rew_std": 431.3969, "Agent": "A2C"}, {"env_step": 270000, "rew": 624.0016, "rew_std": 507.8467, "Agent": "A2C"}, {"env_step": 300000, "rew": 731.1425, "rew_std": 578.3944, "Agent": "A2C"}, {"env_step": 330000, "rew": 796.0141, "rew_std": 644.6127, "Agent": "A2C"}, {"env_step": 360000, "rew": 917.1018, "rew_std": 552.9203, "Agent": "A2C"}, {"env_step": 390000, "rew": 975.4043, "rew_std": 576.3952, "Agent": "A2C"}, {"env_step": 420000, "rew": 934.7208, "rew_std": 561.8865, "Agent": "A2C"}, {"env_step": 450000, "rew": 1119.1488, "rew_std": 647.3409, "Agent": "A2C"}, {"env_step": 480000, "rew": 1154.8508, "rew_std": 658.6049, "Agent": "A2C"}, {"env_step": 510000, "rew": 1076.622, "rew_std": 557.945, "Agent": "A2C"}, {"env_step": 540000, "rew": 1346.3252, "rew_std": 714.3876, "Agent": "A2C"}, {"env_step": 570000, "rew": 1408.3093, "rew_std": 806.4266, "Agent": "A2C"}, {"env_step": 600000, "rew": 1481.7456, "rew_std": 803.4338, "Agent": "A2C"}, {"env_step": 630000, "rew": 1511.0946, "rew_std": 811.9055, "Agent": "A2C"}, {"env_step": 660000, "rew": 1552.3128, "rew_std": 882.5074, "Agent": "A2C"}, {"env_step": 690000, "rew": 1655.6848, "rew_std": 914.6282, "Agent": "A2C"}, {"env_step": 720000, "rew": 1630.8987, "rew_std": 906.7856, "Agent": "A2C"}, {"env_step": 750000, "rew": 1796.7893, "rew_std": 1091.8783, "Agent": "A2C"}, {"env_step": 780000, "rew": 1711.6462, "rew_std": 1041.518, "Agent": "A2C"}, {"env_step": 810000, "rew": 1756.5038, "rew_std": 1036.4954, "Agent": "A2C"}, {"env_step": 840000, "rew": 1764.3135, "rew_std": 1025.8311, "Agent": "A2C"}, {"env_step": 870000, "rew": 1812.3831, "rew_std": 1058.1799, "Agent": "A2C"}, {"env_step": 900000, "rew": 1829.923, "rew_std": 1068.3028, "Agent": "A2C"}, {"env_step": 930000, "rew": 1751.078, "rew_std": 958.1423, "Agent": "A2C"}, {"env_step": 960000, "rew": 1828.2283, "rew_std": 1084.8595, "Agent": "A2C"}, {"env_step": 990000, "rew": 1935.7414, "rew_std": 1159.8274, "Agent": "A2C"}, {"env_step": 1020000, "rew": 1864.5865, "rew_std": 1113.9975, "Agent": "A2C"}, {"env_step": 1050000, "rew": 1881.9668, "rew_std": 1133.5133, "Agent": "A2C"}, {"env_step": 1080000, "rew": 1937.6426, "rew_std": 1153.6651, "Agent": "A2C"}, {"env_step": 1110000, "rew": 1928.2942, "rew_std": 1109.9099, "Agent": "A2C"}, {"env_step": 1140000, "rew": 1924.8877, "rew_std": 1196.4508, "Agent": "A2C"}, {"env_step": 1170000, "rew": 1942.7908, "rew_std": 1204.9327, "Agent": "A2C"}, {"env_step": 1200000, "rew": 1969.991, "rew_std": 1165.5349, "Agent": "A2C"}, {"env_step": 1230000, "rew": 1978.3989, "rew_std": 1163.5834, "Agent": "A2C"}, {"env_step": 1260000, "rew": 1933.0249, "rew_std": 1102.7583, "Agent": "A2C"}, {"env_step": 1290000, "rew": 1939.4646, "rew_std": 1137.9136, "Agent": "A2C"}, {"env_step": 1320000, "rew": 1975.756, "rew_std": 1157.2301, "Agent": "A2C"}, {"env_step": 1350000, "rew": 2043.2366, "rew_std": 1194.5776, "Agent": "A2C"}, {"env_step": 1380000, "rew": 1959.5897, "rew_std": 1145.7464, "Agent": "A2C"}, {"env_step": 1410000, "rew": 1995.257, "rew_std": 1190.176, "Agent": "A2C"}, {"env_step": 1440000, "rew": 1928.3597, "rew_std": 1015.5797, "Agent": "A2C"}, {"env_step": 1470000, "rew": 1892.8145, "rew_std": 1082.5848, "Agent": "A2C"}, {"env_step": 1500000, "rew": 2039.3697, "rew_std": 1250.7594, "Agent": "A2C"}, {"env_step": 1530000, "rew": 2029.4864, "rew_std": 1238.908, "Agent": "A2C"}, {"env_step": 1560000, "rew": 2105.0698, "rew_std": 1217.2523, "Agent": "A2C"}, {"env_step": 1590000, "rew": 2098.116, "rew_std": 1240.7264, "Agent": "A2C"}, {"env_step": 1620000, "rew": 2124.1828, "rew_std": 1284.9803, "Agent": "A2C"}, {"env_step": 1650000, "rew": 2095.9743, "rew_std": 1179.2287, "Agent": "A2C"}, {"env_step": 1680000, "rew": 2141.3761, "rew_std": 1227.2086, "Agent": "A2C"}, {"env_step": 1710000, "rew": 2182.5117, "rew_std": 1272.711, "Agent": "A2C"}, {"env_step": 1740000, "rew": 2117.5139, "rew_std": 1190.7188, "Agent": "A2C"}, {"env_step": 1770000, "rew": 2185.9423, "rew_std": 1307.419, "Agent": "A2C"}, {"env_step": 1800000, "rew": 2220.7673, "rew_std": 1313.4839, "Agent": "A2C"}, {"env_step": 1830000, "rew": 2209.8734, "rew_std": 1277.911, "Agent": "A2C"}, {"env_step": 1860000, "rew": 2169.9106, "rew_std": 1223.3593, "Agent": "A2C"}, {"env_step": 1890000, "rew": 2218.1295, "rew_std": 1282.6223, "Agent": "A2C"}, {"env_step": 1920000, "rew": 2158.5649, "rew_std": 1214.0396, "Agent": "A2C"}, {"env_step": 1950000, "rew": 2183.6557, "rew_std": 1288.1706, "Agent": "A2C"}, {"env_step": 1980000, "rew": 2217.1837, "rew_std": 1306.43, "Agent": "A2C"}, {"env_step": 2010000, "rew": 2231.2067, "rew_std": 1337.9609, "Agent": "A2C"}, {"env_step": 2040000, "rew": 2193.3199, "rew_std": 1228.9301, "Agent": "A2C"}, {"env_step": 2070000, "rew": 2268.7662, "rew_std": 1349.0271, "Agent": "A2C"}, {"env_step": 2100000, "rew": 2264.3721, "rew_std": 1253.8081, "Agent": "A2C"}, {"env_step": 2130000, "rew": 2244.4677, "rew_std": 1302.3581, "Agent": "A2C"}, {"env_step": 2160000, "rew": 2257.2162, "rew_std": 1296.9644, "Agent": "A2C"}, {"env_step": 2190000, "rew": 2181.3572, "rew_std": 1222.576, "Agent": "A2C"}, {"env_step": 2220000, "rew": 2307.0364, "rew_std": 1349.8938, "Agent": "A2C"}, {"env_step": 2250000, "rew": 2262.5205, "rew_std": 1304.059, "Agent": "A2C"}, {"env_step": 2280000, "rew": 2281.8044, "rew_std": 1255.3387, "Agent": "A2C"}, {"env_step": 2310000, "rew": 2298.6342, "rew_std": 1309.8012, "Agent": "A2C"}, {"env_step": 2340000, "rew": 2341.7024, "rew_std": 1321.0374, "Agent": "A2C"}, {"env_step": 2370000, "rew": 2246.4439, "rew_std": 1258.2221, "Agent": "A2C"}, {"env_step": 2400000, "rew": 2229.6385, "rew_std": 1226.8818, "Agent": "A2C"}, {"env_step": 2430000, "rew": 2222.9333, "rew_std": 1238.6857, "Agent": "A2C"}, {"env_step": 2460000, "rew": 2271.9239, "rew_std": 1225.146, "Agent": "A2C"}, {"env_step": 2490000, "rew": 2333.9164, "rew_std": 1287.422, "Agent": "A2C"}, {"env_step": 2520000, "rew": 2212.4533, "rew_std": 1203.0745, "Agent": "A2C"}, {"env_step": 2550000, "rew": 2345.4101, "rew_std": 1304.2363, "Agent": "A2C"}, {"env_step": 2580000, "rew": 2324.3592, "rew_std": 1278.9764, "Agent": "A2C"}, {"env_step": 2610000, "rew": 2329.7909, "rew_std": 1344.4421, "Agent": "A2C"}, {"env_step": 2640000, "rew": 2377.3057, "rew_std": 1363.7383, "Agent": "A2C"}, {"env_step": 2670000, "rew": 2315.1766, "rew_std": 1384.7018, "Agent": "A2C"}, {"env_step": 2700000, "rew": 2343.1761, "rew_std": 1318.4924, "Agent": "A2C"}, {"env_step": 2730000, "rew": 2289.6847, "rew_std": 1298.1297, "Agent": "A2C"}, {"env_step": 2760000, "rew": 2340.3924, "rew_std": 1353.7832, "Agent": "A2C"}, {"env_step": 2790000, "rew": 2301.7639, "rew_std": 1313.5865, "Agent": "A2C"}, {"env_step": 2820000, "rew": 2255.2329, "rew_std": 1275.3453, "Agent": "A2C"}, {"env_step": 2850000, "rew": 2367.6211, "rew_std": 1385.5564, "Agent": "A2C"}, {"env_step": 2880000, "rew": 2270.8572, "rew_std": 1289.8252, "Agent": "A2C"}, {"env_step": 2910000, "rew": 2311.5837, "rew_std": 1262.1833, "Agent": "A2C"}, {"env_step": 2940000, "rew": 2227.0238, "rew_std": 1234.4241, "Agent": "A2C"}, {"env_step": 2970000, "rew": 2285.462, "rew_std": 1210.3531, "Agent": "A2C"}, {"env_step": 3000000, "rew": 2360.4776, "rew_std": 1376.9442, "Agent": "A2C"}] \ No newline at end of file diff --git a/examples/mujoco/benchmark/Hopper-v3/result.json b/examples/mujoco/benchmark/Hopper-v3/result.json new file mode 100644 index 0000000..b48f9cd --- /dev/null +++ b/examples/mujoco/benchmark/Hopper-v3/result.json @@ -0,0 +1 @@ +[{"env_step": 0, "rew": 47.4556, "rew_std": 36.0937, "Agent": "SAC"}, {"env_step": 5000, "rew": 275.7761, "rew_std": 59.4556, "Agent": "SAC"}, {"env_step": 10000, "rew": 282.0917, "rew_std": 46.6158, "Agent": "SAC"}, {"env_step": 15000, "rew": 336.6352, "rew_std": 45.5268, "Agent": "SAC"}, {"env_step": 20000, "rew": 350.7719, "rew_std": 59.4432, "Agent": "SAC"}, {"env_step": 25000, "rew": 393.867, "rew_std": 94.7212, "Agent": "SAC"}, {"env_step": 30000, "rew": 493.3881, "rew_std": 205.9798, "Agent": "SAC"}, {"env_step": 35000, "rew": 535.254, "rew_std": 222.4849, "Agent": "SAC"}, {"env_step": 40000, "rew": 1029.6053, "rew_std": 793.9815, "Agent": "SAC"}, {"env_step": 45000, "rew": 705.2766, "rew_std": 641.1404, "Agent": "SAC"}, {"env_step": 50000, "rew": 851.1729, "rew_std": 372.5027, "Agent": "SAC"}, {"env_step": 55000, "rew": 1182.7687, "rew_std": 721.6847, "Agent": "SAC"}, {"env_step": 60000, "rew": 930.261, "rew_std": 399.8715, "Agent": "SAC"}, {"env_step": 65000, "rew": 1422.8022, "rew_std": 1090.8127, "Agent": "SAC"}, {"env_step": 70000, "rew": 1387.1509, "rew_std": 1089.2428, "Agent": "SAC"}, {"env_step": 75000, "rew": 1521.7049, "rew_std": 1072.2036, "Agent": "SAC"}, {"env_step": 80000, "rew": 1523.1274, "rew_std": 812.4802, "Agent": "SAC"}, {"env_step": 85000, "rew": 1787.7521, "rew_std": 997.5609, "Agent": "SAC"}, {"env_step": 90000, "rew": 1765.9052, "rew_std": 1035.8665, "Agent": "SAC"}, {"env_step": 95000, "rew": 2079.4099, "rew_std": 1125.9664, "Agent": "SAC"}, {"env_step": 100000, "rew": 2613.4592, "rew_std": 990.7811, "Agent": "SAC"}, {"env_step": 105000, "rew": 2505.2093, "rew_std": 1014.8763, "Agent": "SAC"}, {"env_step": 110000, "rew": 1882.9724, "rew_std": 903.8805, "Agent": "SAC"}, {"env_step": 115000, "rew": 2251.6213, "rew_std": 1015.3205, "Agent": "SAC"}, {"env_step": 120000, "rew": 2083.6742, "rew_std": 1150.8883, "Agent": "SAC"}, {"env_step": 125000, "rew": 2037.7974, "rew_std": 1069.9703, "Agent": "SAC"}, {"env_step": 130000, "rew": 2447.5492, "rew_std": 1104.9526, "Agent": "SAC"}, {"env_step": 135000, "rew": 2434.7552, "rew_std": 991.8537, "Agent": "SAC"}, {"env_step": 140000, "rew": 2720.4292, "rew_std": 934.5695, "Agent": "SAC"}, {"env_step": 145000, "rew": 2583.2391, "rew_std": 1002.1706, "Agent": "SAC"}, {"env_step": 150000, "rew": 2782.7663, "rew_std": 827.8125, "Agent": "SAC"}, {"env_step": 155000, "rew": 2899.1311, "rew_std": 683.8363, "Agent": "SAC"}, {"env_step": 160000, "rew": 2912.7348, "rew_std": 704.0542, "Agent": "SAC"}, {"env_step": 165000, "rew": 2908.8992, "rew_std": 758.2095, "Agent": "SAC"}, {"env_step": 170000, "rew": 2830.1015, "rew_std": 582.8762, "Agent": "SAC"}, {"env_step": 175000, "rew": 3058.4135, "rew_std": 416.2314, "Agent": "SAC"}, {"env_step": 180000, "rew": 2800.0321, "rew_std": 969.0885, "Agent": "SAC"}, {"env_step": 185000, "rew": 2993.8637, "rew_std": 707.68, "Agent": "SAC"}, {"env_step": 190000, "rew": 2382.8887, "rew_std": 1161.7076, "Agent": "SAC"}, {"env_step": 195000, "rew": 3015.8118, "rew_std": 506.8064, "Agent": "SAC"}, {"env_step": 200000, "rew": 3234.4156, "rew_std": 314.1973, "Agent": "SAC"}, {"env_step": 205000, "rew": 3099.8164, "rew_std": 619.2829, "Agent": "SAC"}, {"env_step": 210000, "rew": 2495.4626, "rew_std": 1158.0519, "Agent": "SAC"}, {"env_step": 215000, "rew": 2738.6383, "rew_std": 982.2683, "Agent": "SAC"}, {"env_step": 220000, "rew": 2957.1676, "rew_std": 777.4626, "Agent": "SAC"}, {"env_step": 225000, "rew": 2915.008, "rew_std": 750.9862, "Agent": "SAC"}, {"env_step": 230000, "rew": 3173.0933, "rew_std": 545.6744, "Agent": "SAC"}, {"env_step": 235000, "rew": 3352.3255, "rew_std": 144.4022, "Agent": "SAC"}, {"env_step": 240000, "rew": 3170.9914, "rew_std": 657.4997, "Agent": "SAC"}, {"env_step": 245000, "rew": 3294.4914, "rew_std": 278.824, "Agent": "SAC"}, {"env_step": 250000, "rew": 3222.7075, "rew_std": 280.7911, "Agent": "SAC"}, {"env_step": 255000, "rew": 3297.9414, "rew_std": 224.8675, "Agent": "SAC"}, {"env_step": 260000, "rew": 3382.6137, "rew_std": 129.619, "Agent": "SAC"}, {"env_step": 265000, "rew": 3385.8352, "rew_std": 131.3435, "Agent": "SAC"}, {"env_step": 270000, "rew": 3345.2796, "rew_std": 213.9516, "Agent": "SAC"}, {"env_step": 275000, "rew": 3108.6666, "rew_std": 687.6853, "Agent": "SAC"}, {"env_step": 280000, "rew": 2807.6834, "rew_std": 1111.6307, "Agent": "SAC"}, {"env_step": 285000, "rew": 2989.1773, "rew_std": 888.0763, "Agent": "SAC"}, {"env_step": 290000, "rew": 3165.0168, "rew_std": 670.5128, "Agent": "SAC"}, {"env_step": 295000, "rew": 3403.0451, "rew_std": 95.1653, "Agent": "SAC"}, {"env_step": 300000, "rew": 3387.6597, "rew_std": 114.6012, "Agent": "SAC"}, {"env_step": 305000, "rew": 3327.1286, "rew_std": 317.4994, "Agent": "SAC"}, {"env_step": 310000, "rew": 3402.2741, "rew_std": 118.2682, "Agent": "SAC"}, {"env_step": 315000, "rew": 3335.9056, "rew_std": 249.4078, "Agent": "SAC"}, {"env_step": 320000, "rew": 3389.7038, "rew_std": 113.1724, "Agent": "SAC"}, {"env_step": 325000, "rew": 3439.6622, "rew_std": 96.1649, "Agent": "SAC"}, {"env_step": 330000, "rew": 2915.1265, "rew_std": 1128.311, "Agent": "SAC"}, {"env_step": 335000, "rew": 3355.5532, "rew_std": 187.5645, "Agent": "SAC"}, {"env_step": 340000, "rew": 3235.9036, "rew_std": 642.4013, "Agent": "SAC"}, {"env_step": 345000, "rew": 3244.4175, "rew_std": 612.4494, "Agent": "SAC"}, {"env_step": 350000, "rew": 3386.5798, "rew_std": 238.5771, "Agent": "SAC"}, {"env_step": 355000, "rew": 3194.7686, "rew_std": 700.3986, "Agent": "SAC"}, {"env_step": 360000, "rew": 3413.1038, "rew_std": 116.8649, "Agent": "SAC"}, {"env_step": 365000, "rew": 3289.0649, "rew_std": 496.0933, "Agent": "SAC"}, {"env_step": 370000, "rew": 3352.8785, "rew_std": 266.9787, "Agent": "SAC"}, {"env_step": 375000, "rew": 3431.5784, "rew_std": 116.194, "Agent": "SAC"}, {"env_step": 380000, "rew": 3179.7096, "rew_std": 586.7048, "Agent": "SAC"}, {"env_step": 385000, "rew": 3269.7002, "rew_std": 600.8841, "Agent": "SAC"}, {"env_step": 390000, "rew": 3362.5338, "rew_std": 315.9082, "Agent": "SAC"}, {"env_step": 395000, "rew": 3259.2257, "rew_std": 587.5798, "Agent": "SAC"}, {"env_step": 400000, "rew": 3300.7786, "rew_std": 477.0863, "Agent": "SAC"}, {"env_step": 405000, "rew": 3156.5526, "rew_std": 622.6734, "Agent": "SAC"}, {"env_step": 410000, "rew": 3440.1684, "rew_std": 83.916, "Agent": "SAC"}, {"env_step": 415000, "rew": 3427.2914, "rew_std": 112.7821, "Agent": "SAC"}, {"env_step": 420000, "rew": 3278.1821, "rew_std": 417.4718, "Agent": "SAC"}, {"env_step": 425000, "rew": 3286.694, "rew_std": 544.7659, "Agent": "SAC"}, {"env_step": 430000, "rew": 3298.7286, "rew_std": 489.6424, "Agent": "SAC"}, {"env_step": 435000, "rew": 3108.858, "rew_std": 612.0163, "Agent": "SAC"}, {"env_step": 440000, "rew": 3373.391, "rew_std": 303.656, "Agent": "SAC"}, {"env_step": 445000, "rew": 3348.5861, "rew_std": 332.7744, "Agent": "SAC"}, {"env_step": 450000, "rew": 3129.1637, "rew_std": 1045.6309, "Agent": "SAC"}, {"env_step": 455000, "rew": 3315.3106, "rew_std": 446.1883, "Agent": "SAC"}, {"env_step": 460000, "rew": 3351.584, "rew_std": 363.8294, "Agent": "SAC"}, {"env_step": 465000, "rew": 3408.3243, "rew_std": 247.8712, "Agent": "SAC"}, {"env_step": 470000, "rew": 3269.2203, "rew_std": 626.2374, "Agent": "SAC"}, {"env_step": 475000, "rew": 3479.7509, "rew_std": 76.4065, "Agent": "SAC"}, {"env_step": 480000, "rew": 3307.292, "rew_std": 572.615, "Agent": "SAC"}, {"env_step": 485000, "rew": 3358.0348, "rew_std": 392.7433, "Agent": "SAC"}, {"env_step": 490000, "rew": 3289.9174, "rew_std": 620.9423, "Agent": "SAC"}, {"env_step": 495000, "rew": 3149.5783, "rew_std": 983.6098, "Agent": "SAC"}, {"env_step": 500000, "rew": 3153.8445, "rew_std": 689.3463, "Agent": "SAC"}, {"env_step": 505000, "rew": 3343.6685, "rew_std": 428.8474, "Agent": "SAC"}, {"env_step": 510000, "rew": 3352.6351, "rew_std": 453.2007, "Agent": "SAC"}, {"env_step": 515000, "rew": 3496.9786, "rew_std": 67.199, "Agent": "SAC"}, {"env_step": 520000, "rew": 3308.8103, "rew_std": 560.9129, "Agent": "SAC"}, {"env_step": 525000, "rew": 3429.5733, "rew_std": 181.7485, "Agent": "SAC"}, {"env_step": 530000, "rew": 3502.2174, "rew_std": 55.4399, "Agent": "SAC"}, {"env_step": 535000, "rew": 3307.6355, "rew_std": 573.1831, "Agent": "SAC"}, {"env_step": 540000, "rew": 3494.0667, "rew_std": 74.681, "Agent": "SAC"}, {"env_step": 545000, "rew": 3152.4726, "rew_std": 919.3493, "Agent": "SAC"}, {"env_step": 550000, "rew": 3496.8042, "rew_std": 69.283, "Agent": "SAC"}, {"env_step": 555000, "rew": 3472.1212, "rew_std": 103.5854, "Agent": "SAC"}, {"env_step": 560000, "rew": 3466.4866, "rew_std": 110.1099, "Agent": "SAC"}, {"env_step": 565000, "rew": 3485.939, "rew_std": 80.6578, "Agent": "SAC"}, {"env_step": 570000, "rew": 3488.7434, "rew_std": 77.3344, "Agent": "SAC"}, {"env_step": 575000, "rew": 3147.2589, "rew_std": 989.8934, "Agent": "SAC"}, {"env_step": 580000, "rew": 3342.3024, "rew_std": 472.1387, "Agent": "SAC"}, {"env_step": 585000, "rew": 3505.615, "rew_std": 66.4951, "Agent": "SAC"}, {"env_step": 590000, "rew": 3434.079, "rew_std": 222.5851, "Agent": "SAC"}, {"env_step": 595000, "rew": 3302.6204, "rew_std": 527.5602, "Agent": "SAC"}, {"env_step": 600000, "rew": 3469.6485, "rew_std": 129.1016, "Agent": "SAC"}, {"env_step": 605000, "rew": 3453.9831, "rew_std": 128.9461, "Agent": "SAC"}, {"env_step": 610000, "rew": 3496.7596, "rew_std": 49.8348, "Agent": "SAC"}, {"env_step": 615000, "rew": 3497.4815, "rew_std": 62.9814, "Agent": "SAC"}, {"env_step": 620000, "rew": 3509.1201, "rew_std": 45.4985, "Agent": "SAC"}, {"env_step": 625000, "rew": 3513.9445, "rew_std": 49.767, "Agent": "SAC"}, {"env_step": 630000, "rew": 3485.2062, "rew_std": 62.7697, "Agent": "SAC"}, {"env_step": 635000, "rew": 3463.5502, "rew_std": 138.2686, "Agent": "SAC"}, {"env_step": 640000, "rew": 3504.1595, "rew_std": 63.7314, "Agent": "SAC"}, {"env_step": 645000, "rew": 3506.4354, "rew_std": 54.2136, "Agent": "SAC"}, {"env_step": 650000, "rew": 3504.8205, "rew_std": 66.3304, "Agent": "SAC"}, {"env_step": 655000, "rew": 3512.5706, "rew_std": 51.5125, "Agent": "SAC"}, {"env_step": 660000, "rew": 3511.0268, "rew_std": 52.0514, "Agent": "SAC"}, {"env_step": 665000, "rew": 3513.4651, "rew_std": 54.3786, "Agent": "SAC"}, {"env_step": 670000, "rew": 3507.2821, "rew_std": 55.4767, "Agent": "SAC"}, {"env_step": 675000, "rew": 3518.2688, "rew_std": 53.0561, "Agent": "SAC"}, {"env_step": 680000, "rew": 3349.8544, "rew_std": 489.5576, "Agent": "SAC"}, {"env_step": 685000, "rew": 3330.8927, "rew_std": 539.2089, "Agent": "SAC"}, {"env_step": 690000, "rew": 3496.2697, "rew_std": 98.1611, "Agent": "SAC"}, {"env_step": 695000, "rew": 3514.8186, "rew_std": 61.7482, "Agent": "SAC"}, {"env_step": 700000, "rew": 3503.5561, "rew_std": 64.6355, "Agent": "SAC"}, {"env_step": 705000, "rew": 3497.2257, "rew_std": 61.3456, "Agent": "SAC"}, {"env_step": 710000, "rew": 3520.9237, "rew_std": 65.9712, "Agent": "SAC"}, {"env_step": 715000, "rew": 3489.5284, "rew_std": 136.7891, "Agent": "SAC"}, {"env_step": 720000, "rew": 3527.6567, "rew_std": 63.4808, "Agent": "SAC"}, {"env_step": 725000, "rew": 3516.5266, "rew_std": 54.1486, "Agent": "SAC"}, {"env_step": 730000, "rew": 3519.1343, "rew_std": 52.9873, "Agent": "SAC"}, {"env_step": 735000, "rew": 3490.632, "rew_std": 96.6564, "Agent": "SAC"}, {"env_step": 740000, "rew": 3506.4161, "rew_std": 109.7111, "Agent": "SAC"}, {"env_step": 745000, "rew": 3514.3282, "rew_std": 61.8251, "Agent": "SAC"}, {"env_step": 750000, "rew": 3511.3453, "rew_std": 56.5664, "Agent": "SAC"}, {"env_step": 755000, "rew": 3508.2242, "rew_std": 43.2657, "Agent": "SAC"}, {"env_step": 760000, "rew": 3513.4562, "rew_std": 49.6405, "Agent": "SAC"}, {"env_step": 765000, "rew": 3447.2038, "rew_std": 214.2386, "Agent": "SAC"}, {"env_step": 770000, "rew": 3281.5668, "rew_std": 691.0348, "Agent": "SAC"}, {"env_step": 775000, "rew": 3520.0712, "rew_std": 64.6354, "Agent": "SAC"}, {"env_step": 780000, "rew": 3516.2051, "rew_std": 60.5938, "Agent": "SAC"}, {"env_step": 785000, "rew": 3519.0088, "rew_std": 62.0088, "Agent": "SAC"}, {"env_step": 790000, "rew": 3524.6045, "rew_std": 54.7701, "Agent": "SAC"}, {"env_step": 795000, "rew": 3501.6906, "rew_std": 58.4171, "Agent": "SAC"}, {"env_step": 800000, "rew": 3509.3966, "rew_std": 56.6941, "Agent": "SAC"}, {"env_step": 805000, "rew": 3515.611, "rew_std": 54.3113, "Agent": "SAC"}, {"env_step": 810000, "rew": 3522.4115, "rew_std": 60.0603, "Agent": "SAC"}, {"env_step": 815000, "rew": 3525.2187, "rew_std": 60.8522, "Agent": "SAC"}, {"env_step": 820000, "rew": 3517.6993, "rew_std": 60.5311, "Agent": "SAC"}, {"env_step": 825000, "rew": 3511.3608, "rew_std": 53.1438, "Agent": "SAC"}, {"env_step": 830000, "rew": 3525.4017, "rew_std": 59.3785, "Agent": "SAC"}, {"env_step": 835000, "rew": 3517.3174, "rew_std": 51.8875, "Agent": "SAC"}, {"env_step": 840000, "rew": 3511.427, "rew_std": 48.9103, "Agent": "SAC"}, {"env_step": 845000, "rew": 3233.2189, "rew_std": 902.9686, "Agent": "SAC"}, {"env_step": 850000, "rew": 3505.4327, "rew_std": 53.0899, "Agent": "SAC"}, {"env_step": 855000, "rew": 3521.5302, "rew_std": 58.6207, "Agent": "SAC"}, {"env_step": 860000, "rew": 3532.3486, "rew_std": 68.1457, "Agent": "SAC"}, {"env_step": 865000, "rew": 3523.5765, "rew_std": 53.409, "Agent": "SAC"}, {"env_step": 870000, "rew": 3236.7132, "rew_std": 895.7161, "Agent": "SAC"}, {"env_step": 875000, "rew": 3517.0279, "rew_std": 56.7523, "Agent": "SAC"}, {"env_step": 880000, "rew": 3522.4118, "rew_std": 58.5564, "Agent": "SAC"}, {"env_step": 885000, "rew": 3161.5897, "rew_std": 1046.7225, "Agent": "SAC"}, {"env_step": 890000, "rew": 3537.7698, "rew_std": 47.4872, "Agent": "SAC"}, {"env_step": 895000, "rew": 3519.0764, "rew_std": 60.8041, "Agent": "SAC"}, {"env_step": 900000, "rew": 3520.3896, "rew_std": 55.8492, "Agent": "SAC"}, {"env_step": 905000, "rew": 3523.8942, "rew_std": 56.3431, "Agent": "SAC"}, {"env_step": 910000, "rew": 3538.571, "rew_std": 40.6978, "Agent": "SAC"}, {"env_step": 915000, "rew": 3450.8878, "rew_std": 236.0645, "Agent": "SAC"}, {"env_step": 920000, "rew": 3524.2823, "rew_std": 51.914, "Agent": "SAC"}, {"env_step": 925000, "rew": 3202.4734, "rew_std": 977.0959, "Agent": "SAC"}, {"env_step": 930000, "rew": 3273.1453, "rew_std": 734.3886, "Agent": "SAC"}, {"env_step": 935000, "rew": 3455.3452, "rew_std": 206.2434, "Agent": "SAC"}, {"env_step": 940000, "rew": 3175.5728, "rew_std": 1042.4874, "Agent": "SAC"}, {"env_step": 945000, "rew": 3527.0726, "rew_std": 55.3019, "Agent": "SAC"}, {"env_step": 950000, "rew": 3535.6306, "rew_std": 49.2671, "Agent": "SAC"}, {"env_step": 955000, "rew": 3525.8447, "rew_std": 46.4758, "Agent": "SAC"}, {"env_step": 960000, "rew": 3531.017, "rew_std": 52.1814, "Agent": "SAC"}, {"env_step": 965000, "rew": 3520.952, "rew_std": 52.5745, "Agent": "SAC"}, {"env_step": 970000, "rew": 3542.2142, "rew_std": 51.4682, "Agent": "SAC"}, {"env_step": 975000, "rew": 3527.5707, "rew_std": 45.3567, "Agent": "SAC"}, {"env_step": 980000, "rew": 3189.7376, "rew_std": 997.8534, "Agent": "SAC"}, {"env_step": 985000, "rew": 3520.5999, "rew_std": 35.5321, "Agent": "SAC"}, {"env_step": 990000, "rew": 3515.0443, "rew_std": 36.0914, "Agent": "SAC"}, {"env_step": 995000, "rew": 3520.7646, "rew_std": 44.903, "Agent": "SAC"}, {"env_step": 1000000, "rew": 3531.5999, "rew_std": 43.2168, "Agent": "SAC"}, {"env_step": 1005000, "rew": 3537.6764, "rew_std": 41.3989, "Agent": "SAC"}, {"env_step": 1010000, "rew": 3537.5447, "rew_std": 44.6579, "Agent": "SAC"}, {"env_step": 1015000, "rew": 3524.9329, "rew_std": 56.391, "Agent": "SAC"}, {"env_step": 1020000, "rew": 3459.5167, "rew_std": 247.163, "Agent": "SAC"}, {"env_step": 1025000, "rew": 3540.6938, "rew_std": 37.3364, "Agent": "SAC"}, {"env_step": 1030000, "rew": 3533.2482, "rew_std": 48.7726, "Agent": "SAC"}, {"env_step": 1035000, "rew": 3521.0612, "rew_std": 58.0774, "Agent": "SAC"}, {"env_step": 1040000, "rew": 3546.4092, "rew_std": 48.796, "Agent": "SAC"}, {"env_step": 1045000, "rew": 3535.2375, "rew_std": 46.4841, "Agent": "SAC"}, {"env_step": 1050000, "rew": 3546.7265, "rew_std": 42.9841, "Agent": "SAC"}, {"env_step": 1055000, "rew": 3527.3797, "rew_std": 49.7837, "Agent": "SAC"}, {"env_step": 1060000, "rew": 3544.2985, "rew_std": 50.6251, "Agent": "SAC"}, {"env_step": 1065000, "rew": 3504.0597, "rew_std": 137.2038, "Agent": "SAC"}, {"env_step": 1070000, "rew": 3548.3365, "rew_std": 42.6806, "Agent": "SAC"}, {"env_step": 1075000, "rew": 3547.6729, "rew_std": 49.0633, "Agent": "SAC"}, {"env_step": 1080000, "rew": 3546.0549, "rew_std": 40.8882, "Agent": "SAC"}, {"env_step": 1085000, "rew": 3518.5469, "rew_std": 109.8673, "Agent": "SAC"}, {"env_step": 1090000, "rew": 3151.1933, "rew_std": 909.5713, "Agent": "SAC"}, {"env_step": 1095000, "rew": 3440.9245, "rew_std": 322.2829, "Agent": "SAC"}, {"env_step": 1100000, "rew": 3029.8716, "rew_std": 1071.2459, "Agent": "SAC"}, {"env_step": 1105000, "rew": 3557.3477, "rew_std": 33.0738, "Agent": "SAC"}, {"env_step": 1110000, "rew": 3395.5135, "rew_std": 338.1572, "Agent": "SAC"}, {"env_step": 1115000, "rew": 3550.1206, "rew_std": 48.1577, "Agent": "SAC"}, {"env_step": 1120000, "rew": 3561.1023, "rew_std": 40.9386, "Agent": "SAC"}, {"env_step": 1125000, "rew": 3533.8194, "rew_std": 61.1164, "Agent": "SAC"}, {"env_step": 1130000, "rew": 3463.6075, "rew_std": 266.0974, "Agent": "SAC"}, {"env_step": 1135000, "rew": 3570.4994, "rew_std": 47.6836, "Agent": "SAC"}, {"env_step": 1140000, "rew": 3551.0034, "rew_std": 41.5901, "Agent": "SAC"}, {"env_step": 1145000, "rew": 3564.5966, "rew_std": 39.7762, "Agent": "SAC"}, {"env_step": 1150000, "rew": 3181.0511, "rew_std": 1060.7804, "Agent": "SAC"}, {"env_step": 1155000, "rew": 3462.0114, "rew_std": 197.7437, "Agent": "SAC"}, {"env_step": 1160000, "rew": 3425.1199, "rew_std": 361.8834, "Agent": "SAC"}, {"env_step": 1165000, "rew": 3352.6739, "rew_std": 518.6627, "Agent": "SAC"}, {"env_step": 1170000, "rew": 3557.2332, "rew_std": 43.5451, "Agent": "SAC"}, {"env_step": 1175000, "rew": 3554.7394, "rew_std": 35.583, "Agent": "SAC"}, {"env_step": 1180000, "rew": 3557.5383, "rew_std": 31.4509, "Agent": "SAC"}, {"env_step": 1185000, "rew": 3551.5281, "rew_std": 36.8617, "Agent": "SAC"}, {"env_step": 1190000, "rew": 3370.5322, "rew_std": 534.8758, "Agent": "SAC"}, {"env_step": 1195000, "rew": 3462.1363, "rew_std": 255.2329, "Agent": "SAC"}, {"env_step": 1200000, "rew": 3515.0849, "rew_std": 121.4034, "Agent": "SAC"}, {"env_step": 1205000, "rew": 3541.7524, "rew_std": 77.3033, "Agent": "SAC"}, {"env_step": 1210000, "rew": 3567.9682, "rew_std": 50.4551, "Agent": "SAC"}, {"env_step": 1215000, "rew": 3550.3638, "rew_std": 34.3122, "Agent": "SAC"}, {"env_step": 1220000, "rew": 3561.7984, "rew_std": 37.095, "Agent": "SAC"}, {"env_step": 1225000, "rew": 3509.1085, "rew_std": 193.0366, "Agent": "SAC"}, {"env_step": 1230000, "rew": 3531.0292, "rew_std": 99.6654, "Agent": "SAC"}, {"env_step": 1235000, "rew": 3573.5177, "rew_std": 49.6752, "Agent": "SAC"}, {"env_step": 1240000, "rew": 3545.2726, "rew_std": 71.9973, "Agent": "SAC"}, {"env_step": 1245000, "rew": 3487.1151, "rew_std": 227.3082, "Agent": "SAC"}, {"env_step": 1250000, "rew": 3547.1835, "rew_std": 38.5091, "Agent": "SAC"}, {"env_step": 0, "rew": 47.4556, "rew_std": 36.0937, "Agent": "DDPG"}, {"env_step": 5000, "rew": 199.5964, "rew_std": 33.9794, "Agent": "DDPG"}, {"env_step": 10000, "rew": 292.9478, "rew_std": 123.7966, "Agent": "DDPG"}, {"env_step": 15000, "rew": 304.7492, "rew_std": 86.323, "Agent": "DDPG"}, {"env_step": 20000, "rew": 287.3028, "rew_std": 154.7323, "Agent": "DDPG"}, {"env_step": 25000, "rew": 350.8284, "rew_std": 185.3053, "Agent": "DDPG"}, {"env_step": 30000, "rew": 461.0749, "rew_std": 327.0808, "Agent": "DDPG"}, {"env_step": 35000, "rew": 554.0079, "rew_std": 580.9794, "Agent": "DDPG"}, {"env_step": 40000, "rew": 368.9768, "rew_std": 260.8433, "Agent": "DDPG"}, {"env_step": 45000, "rew": 424.1908, "rew_std": 156.5163, "Agent": "DDPG"}, {"env_step": 50000, "rew": 576.619, "rew_std": 305.5415, "Agent": "DDPG"}, {"env_step": 55000, "rew": 491.5706, "rew_std": 285.2554, "Agent": "DDPG"}, {"env_step": 60000, "rew": 395.1139, "rew_std": 248.961, "Agent": "DDPG"}, {"env_step": 65000, "rew": 619.7068, "rew_std": 397.6176, "Agent": "DDPG"}, {"env_step": 70000, "rew": 507.9061, "rew_std": 321.1003, "Agent": "DDPG"}, {"env_step": 75000, "rew": 719.1054, "rew_std": 406.2997, "Agent": "DDPG"}, {"env_step": 80000, "rew": 744.2296, "rew_std": 364.8092, "Agent": "DDPG"}, {"env_step": 85000, "rew": 938.9236, "rew_std": 221.6865, "Agent": "DDPG"}, {"env_step": 90000, "rew": 856.0042, "rew_std": 388.8057, "Agent": "DDPG"}, {"env_step": 95000, "rew": 573.1303, "rew_std": 437.724, "Agent": "DDPG"}, {"env_step": 100000, "rew": 904.4764, "rew_std": 396.2616, "Agent": "DDPG"}, {"env_step": 105000, "rew": 994.7385, "rew_std": 525.0778, "Agent": "DDPG"}, {"env_step": 110000, "rew": 912.3102, "rew_std": 234.5438, "Agent": "DDPG"}, {"env_step": 115000, "rew": 905.7738, "rew_std": 506.0842, "Agent": "DDPG"}, {"env_step": 120000, "rew": 1022.8284, "rew_std": 425.9909, "Agent": "DDPG"}, {"env_step": 125000, "rew": 818.86, "rew_std": 332.6216, "Agent": "DDPG"}, {"env_step": 130000, "rew": 856.2068, "rew_std": 696.1068, "Agent": "DDPG"}, {"env_step": 135000, "rew": 969.3492, "rew_std": 564.7443, "Agent": "DDPG"}, {"env_step": 140000, "rew": 935.8756, "rew_std": 502.7124, "Agent": "DDPG"}, {"env_step": 145000, "rew": 1209.4538, "rew_std": 762.8217, "Agent": "DDPG"}, {"env_step": 150000, "rew": 1071.5299, "rew_std": 668.4726, "Agent": "DDPG"}, {"env_step": 155000, "rew": 1025.5543, "rew_std": 595.0783, "Agent": "DDPG"}, {"env_step": 160000, "rew": 1552.6187, "rew_std": 729.4655, "Agent": "DDPG"}, {"env_step": 165000, "rew": 1143.5506, "rew_std": 769.8467, "Agent": "DDPG"}, {"env_step": 170000, "rew": 1202.557, "rew_std": 625.1311, "Agent": "DDPG"}, {"env_step": 175000, "rew": 1157.8752, "rew_std": 774.7858, "Agent": "DDPG"}, {"env_step": 180000, "rew": 1112.9275, "rew_std": 571.9906, "Agent": "DDPG"}, {"env_step": 185000, "rew": 1294.5619, "rew_std": 751.9106, "Agent": "DDPG"}, {"env_step": 190000, "rew": 1276.959, "rew_std": 616.8828, "Agent": "DDPG"}, {"env_step": 195000, "rew": 1152.8641, "rew_std": 936.5329, "Agent": "DDPG"}, {"env_step": 200000, "rew": 1271.7593, "rew_std": 662.8393, "Agent": "DDPG"}, {"env_step": 205000, "rew": 1386.9656, "rew_std": 864.4552, "Agent": "DDPG"}, {"env_step": 210000, "rew": 1615.4658, "rew_std": 1087.9647, "Agent": "DDPG"}, {"env_step": 215000, "rew": 1485.9, "rew_std": 901.0807, "Agent": "DDPG"}, {"env_step": 220000, "rew": 1233.074, "rew_std": 891.3348, "Agent": "DDPG"}, {"env_step": 225000, "rew": 1495.7882, "rew_std": 599.9693, "Agent": "DDPG"}, {"env_step": 230000, "rew": 1511.5256, "rew_std": 826.3847, "Agent": "DDPG"}, {"env_step": 235000, "rew": 1368.9508, "rew_std": 672.3022, "Agent": "DDPG"}, {"env_step": 240000, "rew": 2055.3458, "rew_std": 796.3992, "Agent": "DDPG"}, {"env_step": 245000, "rew": 1784.053, "rew_std": 868.0014, "Agent": "DDPG"}, {"env_step": 250000, "rew": 1188.9104, "rew_std": 528.9923, "Agent": "DDPG"}, {"env_step": 255000, "rew": 1593.2322, "rew_std": 738.7044, "Agent": "DDPG"}, {"env_step": 260000, "rew": 1558.0799, "rew_std": 565.0878, "Agent": "DDPG"}, {"env_step": 265000, "rew": 1557.7178, "rew_std": 927.1087, "Agent": "DDPG"}, {"env_step": 270000, "rew": 1335.5645, "rew_std": 908.6661, "Agent": "DDPG"}, {"env_step": 275000, "rew": 1699.6929, "rew_std": 745.9329, "Agent": "DDPG"}, {"env_step": 280000, "rew": 1582.9738, "rew_std": 712.9094, "Agent": "DDPG"}, {"env_step": 285000, "rew": 1309.9093, "rew_std": 789.7158, "Agent": "DDPG"}, {"env_step": 290000, "rew": 1612.4216, "rew_std": 907.3729, "Agent": "DDPG"}, {"env_step": 295000, "rew": 1379.8448, "rew_std": 837.7913, "Agent": "DDPG"}, {"env_step": 300000, "rew": 1240.8142, "rew_std": 792.2656, "Agent": "DDPG"}, {"env_step": 305000, "rew": 1603.6851, "rew_std": 788.9357, "Agent": "DDPG"}, {"env_step": 310000, "rew": 1678.6352, "rew_std": 738.9559, "Agent": "DDPG"}, {"env_step": 315000, "rew": 1520.7429, "rew_std": 739.0202, "Agent": "DDPG"}, {"env_step": 320000, "rew": 1341.5378, "rew_std": 811.6886, "Agent": "DDPG"}, {"env_step": 325000, "rew": 1538.3204, "rew_std": 753.9931, "Agent": "DDPG"}, {"env_step": 330000, "rew": 2171.3508, "rew_std": 812.4793, "Agent": "DDPG"}, {"env_step": 335000, "rew": 1597.8247, "rew_std": 504.1572, "Agent": "DDPG"}, {"env_step": 340000, "rew": 1502.7578, "rew_std": 706.8602, "Agent": "DDPG"}, {"env_step": 345000, "rew": 1228.9521, "rew_std": 731.932, "Agent": "DDPG"}, {"env_step": 350000, "rew": 2105.4206, "rew_std": 711.0904, "Agent": "DDPG"}, {"env_step": 355000, "rew": 1222.946, "rew_std": 719.3598, "Agent": "DDPG"}, {"env_step": 360000, "rew": 1446.9002, "rew_std": 874.4786, "Agent": "DDPG"}, {"env_step": 365000, "rew": 1501.5562, "rew_std": 814.7934, "Agent": "DDPG"}, {"env_step": 370000, "rew": 1540.5308, "rew_std": 832.5853, "Agent": "DDPG"}, {"env_step": 375000, "rew": 1618.1915, "rew_std": 985.3355, "Agent": "DDPG"}, {"env_step": 380000, "rew": 1882.0891, "rew_std": 1091.0448, "Agent": "DDPG"}, {"env_step": 385000, "rew": 1645.4651, "rew_std": 735.7729, "Agent": "DDPG"}, {"env_step": 390000, "rew": 1178.6635, "rew_std": 621.3323, "Agent": "DDPG"}, {"env_step": 395000, "rew": 1994.3664, "rew_std": 772.0565, "Agent": "DDPG"}, {"env_step": 400000, "rew": 1854.5865, "rew_std": 891.3537, "Agent": "DDPG"}, {"env_step": 405000, "rew": 1764.6518, "rew_std": 1036.0291, "Agent": "DDPG"}, {"env_step": 410000, "rew": 1484.1159, "rew_std": 1081.0145, "Agent": "DDPG"}, {"env_step": 415000, "rew": 1507.5379, "rew_std": 723.7435, "Agent": "DDPG"}, {"env_step": 420000, "rew": 1282.4193, "rew_std": 759.1662, "Agent": "DDPG"}, {"env_step": 425000, "rew": 1566.2592, "rew_std": 890.281, "Agent": "DDPG"}, {"env_step": 430000, "rew": 1611.8748, "rew_std": 737.8685, "Agent": "DDPG"}, {"env_step": 435000, "rew": 1932.6178, "rew_std": 878.2189, "Agent": "DDPG"}, {"env_step": 440000, "rew": 1861.9749, "rew_std": 712.5401, "Agent": "DDPG"}, {"env_step": 445000, "rew": 1330.4312, "rew_std": 926.2519, "Agent": "DDPG"}, {"env_step": 450000, "rew": 1279.7114, "rew_std": 749.0, "Agent": "DDPG"}, {"env_step": 455000, "rew": 1406.3635, "rew_std": 557.1778, "Agent": "DDPG"}, {"env_step": 460000, "rew": 1813.8324, "rew_std": 788.8594, "Agent": "DDPG"}, {"env_step": 465000, "rew": 1884.2437, "rew_std": 853.1452, "Agent": "DDPG"}, {"env_step": 470000, "rew": 1915.2181, "rew_std": 834.8238, "Agent": "DDPG"}, {"env_step": 475000, "rew": 1080.747, "rew_std": 451.6704, "Agent": "DDPG"}, {"env_step": 480000, "rew": 1358.944, "rew_std": 755.2171, "Agent": "DDPG"}, {"env_step": 485000, "rew": 1161.3047, "rew_std": 466.7425, "Agent": "DDPG"}, {"env_step": 490000, "rew": 1265.8359, "rew_std": 575.7276, "Agent": "DDPG"}, {"env_step": 495000, "rew": 1579.7615, "rew_std": 829.9376, "Agent": "DDPG"}, {"env_step": 500000, "rew": 1937.8091, "rew_std": 749.3357, "Agent": "DDPG"}, {"env_step": 505000, "rew": 1455.1521, "rew_std": 672.3839, "Agent": "DDPG"}, {"env_step": 510000, "rew": 1539.8811, "rew_std": 807.9035, "Agent": "DDPG"}, {"env_step": 515000, "rew": 1667.0292, "rew_std": 1080.7372, "Agent": "DDPG"}, {"env_step": 520000, "rew": 1385.8087, "rew_std": 667.4235, "Agent": "DDPG"}, {"env_step": 525000, "rew": 1362.2604, "rew_std": 915.962, "Agent": "DDPG"}, {"env_step": 530000, "rew": 1858.4627, "rew_std": 1012.5408, "Agent": "DDPG"}, {"env_step": 535000, "rew": 1726.7965, "rew_std": 628.7272, "Agent": "DDPG"}, {"env_step": 540000, "rew": 1776.6725, "rew_std": 898.7636, "Agent": "DDPG"}, {"env_step": 545000, "rew": 1613.2686, "rew_std": 1021.7273, "Agent": "DDPG"}, {"env_step": 550000, "rew": 1372.588, "rew_std": 647.6357, "Agent": "DDPG"}, {"env_step": 555000, "rew": 1666.3868, "rew_std": 821.096, "Agent": "DDPG"}, {"env_step": 560000, "rew": 1207.9626, "rew_std": 684.4657, "Agent": "DDPG"}, {"env_step": 565000, "rew": 1491.7214, "rew_std": 810.4517, "Agent": "DDPG"}, {"env_step": 570000, "rew": 1517.6948, "rew_std": 562.8144, "Agent": "DDPG"}, {"env_step": 575000, "rew": 1419.4688, "rew_std": 634.5265, "Agent": "DDPG"}, {"env_step": 580000, "rew": 1233.782, "rew_std": 695.0035, "Agent": "DDPG"}, {"env_step": 585000, "rew": 1836.1308, "rew_std": 780.2811, "Agent": "DDPG"}, {"env_step": 590000, "rew": 2012.6321, "rew_std": 710.0638, "Agent": "DDPG"}, {"env_step": 595000, "rew": 1827.9922, "rew_std": 844.7668, "Agent": "DDPG"}, {"env_step": 600000, "rew": 1491.2336, "rew_std": 478.5381, "Agent": "DDPG"}, {"env_step": 605000, "rew": 1661.3061, "rew_std": 741.2274, "Agent": "DDPG"}, {"env_step": 610000, "rew": 1913.0473, "rew_std": 901.0005, "Agent": "DDPG"}, {"env_step": 615000, "rew": 1491.7244, "rew_std": 603.3158, "Agent": "DDPG"}, {"env_step": 620000, "rew": 2054.5905, "rew_std": 727.9373, "Agent": "DDPG"}, {"env_step": 625000, "rew": 1717.238, "rew_std": 949.1291, "Agent": "DDPG"}, {"env_step": 630000, "rew": 1453.1425, "rew_std": 734.2967, "Agent": "DDPG"}, {"env_step": 635000, "rew": 1917.0793, "rew_std": 920.3567, "Agent": "DDPG"}, {"env_step": 640000, "rew": 1816.6305, "rew_std": 562.8137, "Agent": "DDPG"}, {"env_step": 645000, "rew": 1350.5139, "rew_std": 762.7955, "Agent": "DDPG"}, {"env_step": 650000, "rew": 1562.9627, "rew_std": 610.504, "Agent": "DDPG"}, {"env_step": 655000, "rew": 1683.4779, "rew_std": 811.7958, "Agent": "DDPG"}, {"env_step": 660000, "rew": 1948.6147, "rew_std": 780.7959, "Agent": "DDPG"}, {"env_step": 665000, "rew": 1502.015, "rew_std": 778.1516, "Agent": "DDPG"}, {"env_step": 670000, "rew": 1888.6714, "rew_std": 890.6181, "Agent": "DDPG"}, {"env_step": 675000, "rew": 1331.4954, "rew_std": 903.252, "Agent": "DDPG"}, {"env_step": 680000, "rew": 1374.4944, "rew_std": 651.6485, "Agent": "DDPG"}, {"env_step": 685000, "rew": 1658.4132, "rew_std": 606.6067, "Agent": "DDPG"}, {"env_step": 690000, "rew": 1834.4839, "rew_std": 835.1019, "Agent": "DDPG"}, {"env_step": 695000, "rew": 1207.13, "rew_std": 629.4921, "Agent": "DDPG"}, {"env_step": 700000, "rew": 1479.8799, "rew_std": 774.4921, "Agent": "DDPG"}, {"env_step": 705000, "rew": 1430.1922, "rew_std": 790.7581, "Agent": "DDPG"}, {"env_step": 710000, "rew": 1148.2108, "rew_std": 469.6791, "Agent": "DDPG"}, {"env_step": 715000, "rew": 1234.3927, "rew_std": 616.3696, "Agent": "DDPG"}, {"env_step": 720000, "rew": 1442.2469, "rew_std": 869.6578, "Agent": "DDPG"}, {"env_step": 725000, "rew": 2047.1622, "rew_std": 997.6007, "Agent": "DDPG"}, {"env_step": 730000, "rew": 1791.2108, "rew_std": 944.3568, "Agent": "DDPG"}, {"env_step": 735000, "rew": 1337.5849, "rew_std": 715.2636, "Agent": "DDPG"}, {"env_step": 740000, "rew": 1440.0945, "rew_std": 375.8207, "Agent": "DDPG"}, {"env_step": 745000, "rew": 1146.3608, "rew_std": 815.5818, "Agent": "DDPG"}, {"env_step": 750000, "rew": 1407.0721, "rew_std": 537.2618, "Agent": "DDPG"}, {"env_step": 755000, "rew": 1880.9328, "rew_std": 1002.6757, "Agent": "DDPG"}, {"env_step": 760000, "rew": 1888.895, "rew_std": 661.2671, "Agent": "DDPG"}, {"env_step": 765000, "rew": 1888.1488, "rew_std": 760.2352, "Agent": "DDPG"}, {"env_step": 770000, "rew": 1648.2798, "rew_std": 780.4931, "Agent": "DDPG"}, {"env_step": 775000, "rew": 1660.9453, "rew_std": 603.4395, "Agent": "DDPG"}, {"env_step": 780000, "rew": 1766.914, "rew_std": 568.0854, "Agent": "DDPG"}, {"env_step": 785000, "rew": 1934.2981, "rew_std": 719.8906, "Agent": "DDPG"}, {"env_step": 790000, "rew": 1251.4397, "rew_std": 638.1787, "Agent": "DDPG"}, {"env_step": 795000, "rew": 1449.0066, "rew_std": 795.1345, "Agent": "DDPG"}, {"env_step": 800000, "rew": 1954.0406, "rew_std": 828.3001, "Agent": "DDPG"}, {"env_step": 805000, "rew": 1768.7785, "rew_std": 858.3872, "Agent": "DDPG"}, {"env_step": 810000, "rew": 2027.9814, "rew_std": 839.1931, "Agent": "DDPG"}, {"env_step": 815000, "rew": 1913.2213, "rew_std": 692.3181, "Agent": "DDPG"}, {"env_step": 820000, "rew": 1599.5942, "rew_std": 844.4248, "Agent": "DDPG"}, {"env_step": 825000, "rew": 1648.2999, "rew_std": 848.1727, "Agent": "DDPG"}, {"env_step": 830000, "rew": 1672.2677, "rew_std": 650.2616, "Agent": "DDPG"}, {"env_step": 835000, "rew": 1426.2038, "rew_std": 603.851, "Agent": "DDPG"}, {"env_step": 840000, "rew": 1201.6071, "rew_std": 713.3597, "Agent": "DDPG"}, {"env_step": 845000, "rew": 1891.6987, "rew_std": 1094.6308, "Agent": "DDPG"}, {"env_step": 850000, "rew": 1831.5032, "rew_std": 788.3598, "Agent": "DDPG"}, {"env_step": 855000, "rew": 1808.393, "rew_std": 811.0075, "Agent": "DDPG"}, {"env_step": 860000, "rew": 1722.02, "rew_std": 897.3865, "Agent": "DDPG"}, {"env_step": 865000, "rew": 1716.2164, "rew_std": 966.9749, "Agent": "DDPG"}, {"env_step": 870000, "rew": 1987.0209, "rew_std": 978.3952, "Agent": "DDPG"}, {"env_step": 875000, "rew": 1737.9764, "rew_std": 832.1536, "Agent": "DDPG"}, {"env_step": 880000, "rew": 1398.2013, "rew_std": 657.9203, "Agent": "DDPG"}, {"env_step": 885000, "rew": 1343.473, "rew_std": 754.4166, "Agent": "DDPG"}, {"env_step": 890000, "rew": 1681.7618, "rew_std": 929.122, "Agent": "DDPG"}, {"env_step": 895000, "rew": 1568.8794, "rew_std": 904.3076, "Agent": "DDPG"}, {"env_step": 900000, "rew": 1601.8665, "rew_std": 488.7929, "Agent": "DDPG"}, {"env_step": 905000, "rew": 1795.0391, "rew_std": 740.0067, "Agent": "DDPG"}, {"env_step": 910000, "rew": 1630.1091, "rew_std": 534.433, "Agent": "DDPG"}, {"env_step": 915000, "rew": 1856.3447, "rew_std": 898.0134, "Agent": "DDPG"}, {"env_step": 920000, "rew": 1773.9826, "rew_std": 892.2861, "Agent": "DDPG"}, {"env_step": 925000, "rew": 1744.7145, "rew_std": 902.0155, "Agent": "DDPG"}, {"env_step": 930000, "rew": 1681.8963, "rew_std": 934.1126, "Agent": "DDPG"}, {"env_step": 935000, "rew": 1544.4927, "rew_std": 869.2729, "Agent": "DDPG"}, {"env_step": 940000, "rew": 1710.695, "rew_std": 892.1599, "Agent": "DDPG"}, {"env_step": 945000, "rew": 1556.9009, "rew_std": 719.2425, "Agent": "DDPG"}, {"env_step": 950000, "rew": 1560.5468, "rew_std": 580.7039, "Agent": "DDPG"}, {"env_step": 955000, "rew": 1648.3686, "rew_std": 822.4474, "Agent": "DDPG"}, {"env_step": 960000, "rew": 2017.7817, "rew_std": 1049.0691, "Agent": "DDPG"}, {"env_step": 965000, "rew": 1799.2743, "rew_std": 804.7937, "Agent": "DDPG"}, {"env_step": 970000, "rew": 1431.8169, "rew_std": 811.7101, "Agent": "DDPG"}, {"env_step": 975000, "rew": 1907.9792, "rew_std": 863.3055, "Agent": "DDPG"}, {"env_step": 980000, "rew": 2151.5975, "rew_std": 920.9613, "Agent": "DDPG"}, {"env_step": 985000, "rew": 1636.355, "rew_std": 932.402, "Agent": "DDPG"}, {"env_step": 990000, "rew": 1931.229, "rew_std": 782.7125, "Agent": "DDPG"}, {"env_step": 995000, "rew": 2197.0024, "rew_std": 971.5598, "Agent": "DDPG"}, {"env_step": 1000000, "rew": 1604.1096, "rew_std": 936.3235, "Agent": "DDPG"}, {"env_step": 1005000, "rew": 1786.0517, "rew_std": 737.9123, "Agent": "DDPG"}, {"env_step": 1010000, "rew": 1909.8342, "rew_std": 871.6584, "Agent": "DDPG"}, {"env_step": 1015000, "rew": 1948.7802, "rew_std": 753.618, "Agent": "DDPG"}, {"env_step": 1020000, "rew": 1893.587, "rew_std": 750.6465, "Agent": "DDPG"}, {"env_step": 1025000, "rew": 1321.526, "rew_std": 862.1018, "Agent": "DDPG"}, {"env_step": 1030000, "rew": 1479.0536, "rew_std": 825.3967, "Agent": "DDPG"}, {"env_step": 1035000, "rew": 1893.2489, "rew_std": 857.862, "Agent": "DDPG"}, {"env_step": 1040000, "rew": 1492.398, "rew_std": 417.7379, "Agent": "DDPG"}, {"env_step": 1045000, "rew": 1968.4006, "rew_std": 1060.8542, "Agent": "DDPG"}, {"env_step": 1050000, "rew": 1699.4541, "rew_std": 693.4526, "Agent": "DDPG"}, {"env_step": 1055000, "rew": 1486.8643, "rew_std": 608.8139, "Agent": "DDPG"}, {"env_step": 1060000, "rew": 1834.2127, "rew_std": 806.547, "Agent": "DDPG"}, {"env_step": 1065000, "rew": 1898.1982, "rew_std": 868.8335, "Agent": "DDPG"}, {"env_step": 1070000, "rew": 1684.3683, "rew_std": 1056.3452, "Agent": "DDPG"}, {"env_step": 1075000, "rew": 1486.0796, "rew_std": 675.3654, "Agent": "DDPG"}, {"env_step": 1080000, "rew": 1803.2819, "rew_std": 650.8471, "Agent": "DDPG"}, {"env_step": 1085000, "rew": 1857.7992, "rew_std": 712.9232, "Agent": "DDPG"}, {"env_step": 1090000, "rew": 1920.1208, "rew_std": 741.0524, "Agent": "DDPG"}, {"env_step": 1095000, "rew": 1527.1626, "rew_std": 612.0535, "Agent": "DDPG"}, {"env_step": 1100000, "rew": 1914.8211, "rew_std": 878.0446, "Agent": "DDPG"}, {"env_step": 1105000, "rew": 1938.9422, "rew_std": 845.4695, "Agent": "DDPG"}, {"env_step": 1110000, "rew": 1898.7433, "rew_std": 778.8337, "Agent": "DDPG"}, {"env_step": 1115000, "rew": 1700.4836, "rew_std": 728.2356, "Agent": "DDPG"}, {"env_step": 1120000, "rew": 1972.3923, "rew_std": 724.0617, "Agent": "DDPG"}, {"env_step": 1125000, "rew": 1285.6144, "rew_std": 738.7202, "Agent": "DDPG"}, {"env_step": 1130000, "rew": 1615.0117, "rew_std": 726.2691, "Agent": "DDPG"}, {"env_step": 1135000, "rew": 1819.1191, "rew_std": 986.4788, "Agent": "DDPG"}, {"env_step": 1140000, "rew": 1942.4306, "rew_std": 760.4511, "Agent": "DDPG"}, {"env_step": 1145000, "rew": 1221.439, "rew_std": 556.0734, "Agent": "DDPG"}, {"env_step": 1150000, "rew": 1531.2899, "rew_std": 777.4329, "Agent": "DDPG"}, {"env_step": 1155000, "rew": 1326.1052, "rew_std": 652.3935, "Agent": "DDPG"}, {"env_step": 1160000, "rew": 1602.6076, "rew_std": 1082.0088, "Agent": "DDPG"}, {"env_step": 1165000, "rew": 1315.7433, "rew_std": 1057.8408, "Agent": "DDPG"}, {"env_step": 1170000, "rew": 1335.9779, "rew_std": 861.6155, "Agent": "DDPG"}, {"env_step": 1175000, "rew": 1171.8838, "rew_std": 994.2239, "Agent": "DDPG"}, {"env_step": 1180000, "rew": 1324.1715, "rew_std": 500.5259, "Agent": "DDPG"}, {"env_step": 1185000, "rew": 1494.8449, "rew_std": 781.0521, "Agent": "DDPG"}, {"env_step": 1190000, "rew": 1789.3493, "rew_std": 1027.1175, "Agent": "DDPG"}, {"env_step": 1195000, "rew": 1712.5583, "rew_std": 977.8344, "Agent": "DDPG"}, {"env_step": 1200000, "rew": 1564.3719, "rew_std": 649.959, "Agent": "DDPG"}, {"env_step": 1205000, "rew": 1433.258, "rew_std": 548.2289, "Agent": "DDPG"}, {"env_step": 1210000, "rew": 1008.6215, "rew_std": 401.7016, "Agent": "DDPG"}, {"env_step": 1215000, "rew": 1330.1364, "rew_std": 766.7963, "Agent": "DDPG"}, {"env_step": 1220000, "rew": 1759.6604, "rew_std": 909.4543, "Agent": "DDPG"}, {"env_step": 1225000, "rew": 1995.3988, "rew_std": 1042.6365, "Agent": "DDPG"}, {"env_step": 1230000, "rew": 1585.5809, "rew_std": 919.0067, "Agent": "DDPG"}, {"env_step": 1235000, "rew": 1708.5545, "rew_std": 1047.1052, "Agent": "DDPG"}, {"env_step": 1240000, "rew": 1646.8702, "rew_std": 992.2616, "Agent": "DDPG"}, {"env_step": 1245000, "rew": 1340.428, "rew_std": 824.6133, "Agent": "DDPG"}, {"env_step": 1250000, "rew": 1353.3964, "rew_std": 823.2441, "Agent": "DDPG"}, {"env_step": 0, "rew": 47.4556, "rew_std": 36.0937, "Agent": "TD3"}, {"env_step": 5000, "rew": 181.1706, "rew_std": 11.8607, "Agent": "TD3"}, {"env_step": 10000, "rew": 195.6339, "rew_std": 28.1807, "Agent": "TD3"}, {"env_step": 15000, "rew": 196.3426, "rew_std": 38.924, "Agent": "TD3"}, {"env_step": 20000, "rew": 245.9809, "rew_std": 64.7713, "Agent": "TD3"}, {"env_step": 25000, "rew": 226.7399, "rew_std": 45.5801, "Agent": "TD3"}, {"env_step": 30000, "rew": 244.7301, "rew_std": 37.4813, "Agent": "TD3"}, {"env_step": 35000, "rew": 230.0412, "rew_std": 31.198, "Agent": "TD3"}, {"env_step": 40000, "rew": 261.6885, "rew_std": 26.8584, "Agent": "TD3"}, {"env_step": 45000, "rew": 329.7057, "rew_std": 58.7719, "Agent": "TD3"}, {"env_step": 50000, "rew": 321.4114, "rew_std": 45.1645, "Agent": "TD3"}, {"env_step": 55000, "rew": 403.0395, "rew_std": 213.6362, "Agent": "TD3"}, {"env_step": 60000, "rew": 341.5308, "rew_std": 98.1405, "Agent": "TD3"}, {"env_step": 65000, "rew": 461.4619, "rew_std": 232.873, "Agent": "TD3"}, {"env_step": 70000, "rew": 346.6516, "rew_std": 94.5479, "Agent": "TD3"}, {"env_step": 75000, "rew": 336.0042, "rew_std": 81.4701, "Agent": "TD3"}, {"env_step": 80000, "rew": 391.4512, "rew_std": 149.1327, "Agent": "TD3"}, {"env_step": 85000, "rew": 544.4272, "rew_std": 227.6467, "Agent": "TD3"}, {"env_step": 90000, "rew": 710.7108, "rew_std": 549.5041, "Agent": "TD3"}, {"env_step": 95000, "rew": 500.0801, "rew_std": 247.3054, "Agent": "TD3"}, {"env_step": 100000, "rew": 596.5279, "rew_std": 287.8254, "Agent": "TD3"}, {"env_step": 105000, "rew": 623.8025, "rew_std": 327.5888, "Agent": "TD3"}, {"env_step": 110000, "rew": 513.4111, "rew_std": 255.0535, "Agent": "TD3"}, {"env_step": 115000, "rew": 495.5632, "rew_std": 212.4063, "Agent": "TD3"}, {"env_step": 120000, "rew": 555.7409, "rew_std": 227.1517, "Agent": "TD3"}, {"env_step": 125000, "rew": 718.938, "rew_std": 723.5998, "Agent": "TD3"}, {"env_step": 130000, "rew": 680.9442, "rew_std": 460.8098, "Agent": "TD3"}, {"env_step": 135000, "rew": 520.8898, "rew_std": 214.5221, "Agent": "TD3"}, {"env_step": 140000, "rew": 650.3446, "rew_std": 461.5131, "Agent": "TD3"}, {"env_step": 145000, "rew": 684.433, "rew_std": 475.0653, "Agent": "TD3"}, {"env_step": 150000, "rew": 696.6079, "rew_std": 409.8031, "Agent": "TD3"}, {"env_step": 155000, "rew": 927.7476, "rew_std": 772.3126, "Agent": "TD3"}, {"env_step": 160000, "rew": 625.1645, "rew_std": 477.6126, "Agent": "TD3"}, {"env_step": 165000, "rew": 794.7899, "rew_std": 506.2143, "Agent": "TD3"}, {"env_step": 170000, "rew": 1005.2103, "rew_std": 853.9211, "Agent": "TD3"}, {"env_step": 175000, "rew": 845.2076, "rew_std": 627.2596, "Agent": "TD3"}, {"env_step": 180000, "rew": 854.2243, "rew_std": 463.2582, "Agent": "TD3"}, {"env_step": 185000, "rew": 882.8193, "rew_std": 611.5015, "Agent": "TD3"}, {"env_step": 190000, "rew": 881.529, "rew_std": 520.6295, "Agent": "TD3"}, {"env_step": 195000, "rew": 936.7988, "rew_std": 578.7772, "Agent": "TD3"}, {"env_step": 200000, "rew": 887.1944, "rew_std": 649.2743, "Agent": "TD3"}, {"env_step": 205000, "rew": 1003.2064, "rew_std": 719.9525, "Agent": "TD3"}, {"env_step": 210000, "rew": 1259.7631, "rew_std": 918.6324, "Agent": "TD3"}, {"env_step": 215000, "rew": 1218.8341, "rew_std": 909.5814, "Agent": "TD3"}, {"env_step": 220000, "rew": 1454.8435, "rew_std": 961.2179, "Agent": "TD3"}, {"env_step": 225000, "rew": 1120.2673, "rew_std": 742.6399, "Agent": "TD3"}, {"env_step": 230000, "rew": 1388.694, "rew_std": 971.8594, "Agent": "TD3"}, {"env_step": 235000, "rew": 1141.3244, "rew_std": 584.5602, "Agent": "TD3"}, {"env_step": 240000, "rew": 1218.6979, "rew_std": 672.9531, "Agent": "TD3"}, {"env_step": 245000, "rew": 1227.6752, "rew_std": 779.683, "Agent": "TD3"}, {"env_step": 250000, "rew": 1426.1448, "rew_std": 883.1093, "Agent": "TD3"}, {"env_step": 255000, "rew": 1356.3186, "rew_std": 913.06, "Agent": "TD3"}, {"env_step": 260000, "rew": 1519.809, "rew_std": 974.498, "Agent": "TD3"}, {"env_step": 265000, "rew": 1430.8821, "rew_std": 1039.3051, "Agent": "TD3"}, {"env_step": 270000, "rew": 1697.1211, "rew_std": 967.8384, "Agent": "TD3"}, {"env_step": 275000, "rew": 1505.3005, "rew_std": 877.8138, "Agent": "TD3"}, {"env_step": 280000, "rew": 1616.8995, "rew_std": 1006.4543, "Agent": "TD3"}, {"env_step": 285000, "rew": 1595.6735, "rew_std": 880.8836, "Agent": "TD3"}, {"env_step": 290000, "rew": 1669.8616, "rew_std": 788.7447, "Agent": "TD3"}, {"env_step": 295000, "rew": 1680.3393, "rew_std": 771.5441, "Agent": "TD3"}, {"env_step": 300000, "rew": 1583.401, "rew_std": 639.2439, "Agent": "TD3"}, {"env_step": 305000, "rew": 1410.9341, "rew_std": 802.8923, "Agent": "TD3"}, {"env_step": 310000, "rew": 1579.7909, "rew_std": 833.9381, "Agent": "TD3"}, {"env_step": 315000, "rew": 1790.2155, "rew_std": 852.3138, "Agent": "TD3"}, {"env_step": 320000, "rew": 1725.0626, "rew_std": 1004.9677, "Agent": "TD3"}, {"env_step": 325000, "rew": 1773.9219, "rew_std": 818.0885, "Agent": "TD3"}, {"env_step": 330000, "rew": 1934.2742, "rew_std": 983.5329, "Agent": "TD3"}, {"env_step": 335000, "rew": 2059.7185, "rew_std": 992.11, "Agent": "TD3"}, {"env_step": 340000, "rew": 2165.3606, "rew_std": 801.6612, "Agent": "TD3"}, {"env_step": 345000, "rew": 2249.709, "rew_std": 867.5837, "Agent": "TD3"}, {"env_step": 350000, "rew": 2394.7054, "rew_std": 793.9618, "Agent": "TD3"}, {"env_step": 355000, "rew": 2366.3303, "rew_std": 864.5402, "Agent": "TD3"}, {"env_step": 360000, "rew": 2160.4499, "rew_std": 615.6932, "Agent": "TD3"}, {"env_step": 365000, "rew": 2308.3733, "rew_std": 745.4213, "Agent": "TD3"}, {"env_step": 370000, "rew": 1827.3007, "rew_std": 889.1223, "Agent": "TD3"}, {"env_step": 375000, "rew": 2541.1833, "rew_std": 828.5518, "Agent": "TD3"}, {"env_step": 380000, "rew": 2423.2221, "rew_std": 908.2991, "Agent": "TD3"}, {"env_step": 385000, "rew": 2213.5402, "rew_std": 885.9217, "Agent": "TD3"}, {"env_step": 390000, "rew": 2481.258, "rew_std": 730.3209, "Agent": "TD3"}, {"env_step": 395000, "rew": 2167.8909, "rew_std": 919.1415, "Agent": "TD3"}, {"env_step": 400000, "rew": 2247.8712, "rew_std": 1008.3715, "Agent": "TD3"}, {"env_step": 405000, "rew": 2381.9554, "rew_std": 942.0726, "Agent": "TD3"}, {"env_step": 410000, "rew": 2653.6324, "rew_std": 858.8246, "Agent": "TD3"}, {"env_step": 415000, "rew": 2547.441, "rew_std": 933.1938, "Agent": "TD3"}, {"env_step": 420000, "rew": 2613.1125, "rew_std": 735.4271, "Agent": "TD3"}, {"env_step": 425000, "rew": 2594.5911, "rew_std": 757.9404, "Agent": "TD3"}, {"env_step": 430000, "rew": 2264.0453, "rew_std": 1073.4573, "Agent": "TD3"}, {"env_step": 435000, "rew": 2646.8585, "rew_std": 896.8083, "Agent": "TD3"}, {"env_step": 440000, "rew": 2755.1494, "rew_std": 610.0019, "Agent": "TD3"}, {"env_step": 445000, "rew": 2805.4777, "rew_std": 803.6409, "Agent": "TD3"}, {"env_step": 450000, "rew": 2875.6351, "rew_std": 744.3018, "Agent": "TD3"}, {"env_step": 455000, "rew": 2817.7326, "rew_std": 794.6529, "Agent": "TD3"}, {"env_step": 460000, "rew": 2793.4703, "rew_std": 717.5994, "Agent": "TD3"}, {"env_step": 465000, "rew": 2842.3719, "rew_std": 765.139, "Agent": "TD3"}, {"env_step": 470000, "rew": 2563.8287, "rew_std": 712.3143, "Agent": "TD3"}, {"env_step": 475000, "rew": 2669.7186, "rew_std": 933.7879, "Agent": "TD3"}, {"env_step": 480000, "rew": 2685.2956, "rew_std": 968.0261, "Agent": "TD3"}, {"env_step": 485000, "rew": 2700.6731, "rew_std": 821.3018, "Agent": "TD3"}, {"env_step": 490000, "rew": 2964.1087, "rew_std": 448.9934, "Agent": "TD3"}, {"env_step": 495000, "rew": 2943.0135, "rew_std": 647.4103, "Agent": "TD3"}, {"env_step": 500000, "rew": 2961.4207, "rew_std": 655.0678, "Agent": "TD3"}, {"env_step": 505000, "rew": 2849.137, "rew_std": 761.0781, "Agent": "TD3"}, {"env_step": 510000, "rew": 3098.253, "rew_std": 469.053, "Agent": "TD3"}, {"env_step": 515000, "rew": 3154.2582, "rew_std": 501.1539, "Agent": "TD3"}, {"env_step": 520000, "rew": 2987.3938, "rew_std": 511.4157, "Agent": "TD3"}, {"env_step": 525000, "rew": 3201.9522, "rew_std": 309.5776, "Agent": "TD3"}, {"env_step": 530000, "rew": 3127.0868, "rew_std": 499.4894, "Agent": "TD3"}, {"env_step": 535000, "rew": 3231.7025, "rew_std": 247.2752, "Agent": "TD3"}, {"env_step": 540000, "rew": 3246.6948, "rew_std": 423.8079, "Agent": "TD3"}, {"env_step": 545000, "rew": 3244.2828, "rew_std": 285.0854, "Agent": "TD3"}, {"env_step": 550000, "rew": 3296.3845, "rew_std": 280.0152, "Agent": "TD3"}, {"env_step": 555000, "rew": 3339.8197, "rew_std": 133.4885, "Agent": "TD3"}, {"env_step": 560000, "rew": 3292.3128, "rew_std": 240.2349, "Agent": "TD3"}, {"env_step": 565000, "rew": 3351.0744, "rew_std": 136.4845, "Agent": "TD3"}, {"env_step": 570000, "rew": 3253.555, "rew_std": 283.2193, "Agent": "TD3"}, {"env_step": 575000, "rew": 3142.2054, "rew_std": 641.4588, "Agent": "TD3"}, {"env_step": 580000, "rew": 3145.471, "rew_std": 655.4939, "Agent": "TD3"}, {"env_step": 585000, "rew": 3212.4539, "rew_std": 238.0762, "Agent": "TD3"}, {"env_step": 590000, "rew": 3020.1885, "rew_std": 722.6471, "Agent": "TD3"}, {"env_step": 595000, "rew": 3354.9395, "rew_std": 97.4237, "Agent": "TD3"}, {"env_step": 600000, "rew": 3192.8687, "rew_std": 478.9294, "Agent": "TD3"}, {"env_step": 605000, "rew": 2998.3348, "rew_std": 798.7227, "Agent": "TD3"}, {"env_step": 610000, "rew": 3357.0238, "rew_std": 134.9546, "Agent": "TD3"}, {"env_step": 615000, "rew": 3366.7968, "rew_std": 103.0748, "Agent": "TD3"}, {"env_step": 620000, "rew": 3206.5507, "rew_std": 436.868, "Agent": "TD3"}, {"env_step": 625000, "rew": 3171.6245, "rew_std": 659.9982, "Agent": "TD3"}, {"env_step": 630000, "rew": 3231.2786, "rew_std": 265.3576, "Agent": "TD3"}, {"env_step": 635000, "rew": 3307.9242, "rew_std": 241.2563, "Agent": "TD3"}, {"env_step": 640000, "rew": 3016.9011, "rew_std": 926.3848, "Agent": "TD3"}, {"env_step": 645000, "rew": 3368.7732, "rew_std": 129.5799, "Agent": "TD3"}, {"env_step": 650000, "rew": 3227.2273, "rew_std": 289.7332, "Agent": "TD3"}, {"env_step": 655000, "rew": 3240.2439, "rew_std": 365.6094, "Agent": "TD3"}, {"env_step": 660000, "rew": 3227.2667, "rew_std": 285.7852, "Agent": "TD3"}, {"env_step": 665000, "rew": 3265.8132, "rew_std": 418.3806, "Agent": "TD3"}, {"env_step": 670000, "rew": 3282.3655, "rew_std": 368.4049, "Agent": "TD3"}, {"env_step": 675000, "rew": 3258.526, "rew_std": 377.9988, "Agent": "TD3"}, {"env_step": 680000, "rew": 3263.2283, "rew_std": 152.4962, "Agent": "TD3"}, {"env_step": 685000, "rew": 3089.7907, "rew_std": 295.5492, "Agent": "TD3"}, {"env_step": 690000, "rew": 3172.968, "rew_std": 315.6106, "Agent": "TD3"}, {"env_step": 695000, "rew": 3092.3138, "rew_std": 482.2803, "Agent": "TD3"}, {"env_step": 700000, "rew": 3086.4148, "rew_std": 537.6133, "Agent": "TD3"}, {"env_step": 705000, "rew": 3187.0369, "rew_std": 376.8388, "Agent": "TD3"}, {"env_step": 710000, "rew": 3257.3183, "rew_std": 384.8109, "Agent": "TD3"}, {"env_step": 715000, "rew": 3217.0255, "rew_std": 331.0033, "Agent": "TD3"}, {"env_step": 720000, "rew": 3053.4691, "rew_std": 728.2795, "Agent": "TD3"}, {"env_step": 725000, "rew": 3255.4324, "rew_std": 315.9673, "Agent": "TD3"}, {"env_step": 730000, "rew": 3048.3769, "rew_std": 760.9075, "Agent": "TD3"}, {"env_step": 735000, "rew": 3265.5531, "rew_std": 269.5899, "Agent": "TD3"}, {"env_step": 740000, "rew": 3283.9958, "rew_std": 289.6054, "Agent": "TD3"}, {"env_step": 745000, "rew": 3073.2362, "rew_std": 408.9806, "Agent": "TD3"}, {"env_step": 750000, "rew": 3045.4661, "rew_std": 593.2175, "Agent": "TD3"}, {"env_step": 755000, "rew": 3077.6696, "rew_std": 472.3235, "Agent": "TD3"}, {"env_step": 760000, "rew": 3080.2428, "rew_std": 695.1234, "Agent": "TD3"}, {"env_step": 765000, "rew": 3198.3011, "rew_std": 231.1821, "Agent": "TD3"}, {"env_step": 770000, "rew": 3342.6777, "rew_std": 303.2243, "Agent": "TD3"}, {"env_step": 775000, "rew": 3374.2952, "rew_std": 204.2809, "Agent": "TD3"}, {"env_step": 780000, "rew": 3350.8202, "rew_std": 251.5599, "Agent": "TD3"}, {"env_step": 785000, "rew": 3417.5119, "rew_std": 153.9788, "Agent": "TD3"}, {"env_step": 790000, "rew": 3062.315, "rew_std": 753.7818, "Agent": "TD3"}, {"env_step": 795000, "rew": 3443.6579, "rew_std": 140.8264, "Agent": "TD3"}, {"env_step": 800000, "rew": 3214.8637, "rew_std": 393.858, "Agent": "TD3"}, {"env_step": 805000, "rew": 3172.6333, "rew_std": 538.8215, "Agent": "TD3"}, {"env_step": 810000, "rew": 3314.041, "rew_std": 291.0874, "Agent": "TD3"}, {"env_step": 815000, "rew": 3414.3737, "rew_std": 133.3112, "Agent": "TD3"}, {"env_step": 820000, "rew": 2948.3976, "rew_std": 1085.3824, "Agent": "TD3"}, {"env_step": 825000, "rew": 3418.1071, "rew_std": 219.9899, "Agent": "TD3"}, {"env_step": 830000, "rew": 3054.1659, "rew_std": 749.0992, "Agent": "TD3"}, {"env_step": 835000, "rew": 3421.8961, "rew_std": 174.2378, "Agent": "TD3"}, {"env_step": 840000, "rew": 3402.1026, "rew_std": 254.3867, "Agent": "TD3"}, {"env_step": 845000, "rew": 3391.7669, "rew_std": 209.177, "Agent": "TD3"}, {"env_step": 850000, "rew": 3122.6524, "rew_std": 611.6579, "Agent": "TD3"}, {"env_step": 855000, "rew": 3437.683, "rew_std": 156.0111, "Agent": "TD3"}, {"env_step": 860000, "rew": 3421.7968, "rew_std": 175.8982, "Agent": "TD3"}, {"env_step": 865000, "rew": 3472.2399, "rew_std": 116.7905, "Agent": "TD3"}, {"env_step": 870000, "rew": 3181.0626, "rew_std": 532.352, "Agent": "TD3"}, {"env_step": 875000, "rew": 3355.169, "rew_std": 366.8383, "Agent": "TD3"}, {"env_step": 880000, "rew": 3276.6772, "rew_std": 365.8959, "Agent": "TD3"}, {"env_step": 885000, "rew": 3234.2254, "rew_std": 380.1403, "Agent": "TD3"}, {"env_step": 890000, "rew": 3449.26, "rew_std": 159.677, "Agent": "TD3"}, {"env_step": 895000, "rew": 3092.4727, "rew_std": 723.0347, "Agent": "TD3"}, {"env_step": 900000, "rew": 3378.2511, "rew_std": 174.6047, "Agent": "TD3"}, {"env_step": 905000, "rew": 3419.4817, "rew_std": 97.2881, "Agent": "TD3"}, {"env_step": 910000, "rew": 3385.7236, "rew_std": 231.5133, "Agent": "TD3"}, {"env_step": 915000, "rew": 3168.5126, "rew_std": 752.5883, "Agent": "TD3"}, {"env_step": 920000, "rew": 3258.2632, "rew_std": 564.3516, "Agent": "TD3"}, {"env_step": 925000, "rew": 3328.8538, "rew_std": 287.7357, "Agent": "TD3"}, {"env_step": 930000, "rew": 3416.1365, "rew_std": 199.7163, "Agent": "TD3"}, {"env_step": 935000, "rew": 3310.5707, "rew_std": 422.2578, "Agent": "TD3"}, {"env_step": 940000, "rew": 3469.4841, "rew_std": 132.6303, "Agent": "TD3"}, {"env_step": 945000, "rew": 3411.9133, "rew_std": 184.6796, "Agent": "TD3"}, {"env_step": 950000, "rew": 3445.803, "rew_std": 188.1911, "Agent": "TD3"}, {"env_step": 955000, "rew": 3433.6352, "rew_std": 199.4321, "Agent": "TD3"}, {"env_step": 960000, "rew": 3252.8105, "rew_std": 695.7155, "Agent": "TD3"}, {"env_step": 965000, "rew": 3229.0901, "rew_std": 685.0382, "Agent": "TD3"}, {"env_step": 970000, "rew": 3035.6979, "rew_std": 1069.2395, "Agent": "TD3"}, {"env_step": 975000, "rew": 3298.0468, "rew_std": 460.4312, "Agent": "TD3"}, {"env_step": 980000, "rew": 3349.4871, "rew_std": 261.0919, "Agent": "TD3"}, {"env_step": 985000, "rew": 3318.7927, "rew_std": 490.6575, "Agent": "TD3"}, {"env_step": 990000, "rew": 3055.1773, "rew_std": 695.5763, "Agent": "TD3"}, {"env_step": 995000, "rew": 3264.6276, "rew_std": 440.8577, "Agent": "TD3"}, {"env_step": 1000000, "rew": 3132.5766, "rew_std": 771.7353, "Agent": "TD3"}, {"env_step": 1005000, "rew": 3309.2497, "rew_std": 546.4882, "Agent": "TD3"}, {"env_step": 1010000, "rew": 3308.7683, "rew_std": 358.811, "Agent": "TD3"}, {"env_step": 1015000, "rew": 3300.2122, "rew_std": 407.792, "Agent": "TD3"}, {"env_step": 1020000, "rew": 3454.3862, "rew_std": 180.6148, "Agent": "TD3"}, {"env_step": 1025000, "rew": 3355.6457, "rew_std": 306.9581, "Agent": "TD3"}, {"env_step": 1030000, "rew": 3362.1583, "rew_std": 296.1113, "Agent": "TD3"}, {"env_step": 1035000, "rew": 3398.9778, "rew_std": 186.8083, "Agent": "TD3"}, {"env_step": 1040000, "rew": 3388.1484, "rew_std": 205.7846, "Agent": "TD3"}, {"env_step": 1045000, "rew": 3487.3008, "rew_std": 116.211, "Agent": "TD3"}, {"env_step": 1050000, "rew": 3451.232, "rew_std": 110.6646, "Agent": "TD3"}, {"env_step": 1055000, "rew": 3088.9652, "rew_std": 879.486, "Agent": "TD3"}, {"env_step": 1060000, "rew": 3363.9858, "rew_std": 303.7124, "Agent": "TD3"}, {"env_step": 1065000, "rew": 3415.9527, "rew_std": 241.6345, "Agent": "TD3"}, {"env_step": 1070000, "rew": 3220.853, "rew_std": 626.2711, "Agent": "TD3"}, {"env_step": 1075000, "rew": 3387.6881, "rew_std": 241.4848, "Agent": "TD3"}, {"env_step": 1080000, "rew": 3492.6301, "rew_std": 107.173, "Agent": "TD3"}, {"env_step": 1085000, "rew": 3496.5603, "rew_std": 90.9081, "Agent": "TD3"}, {"env_step": 1090000, "rew": 3497.1501, "rew_std": 100.4975, "Agent": "TD3"}, {"env_step": 1095000, "rew": 3245.6992, "rew_std": 528.5044, "Agent": "TD3"}, {"env_step": 1100000, "rew": 3214.6306, "rew_std": 509.1668, "Agent": "TD3"}, {"env_step": 1105000, "rew": 3470.5285, "rew_std": 116.7226, "Agent": "TD3"}, {"env_step": 1110000, "rew": 3495.2515, "rew_std": 158.7062, "Agent": "TD3"}, {"env_step": 1115000, "rew": 3478.5048, "rew_std": 144.9464, "Agent": "TD3"}, {"env_step": 1120000, "rew": 3281.0796, "rew_std": 527.6014, "Agent": "TD3"}, {"env_step": 1125000, "rew": 3192.937, "rew_std": 669.8894, "Agent": "TD3"}, {"env_step": 1130000, "rew": 3438.1592, "rew_std": 230.622, "Agent": "TD3"}, {"env_step": 1135000, "rew": 3334.115, "rew_std": 460.8531, "Agent": "TD3"}, {"env_step": 1140000, "rew": 3071.3359, "rew_std": 817.2291, "Agent": "TD3"}, {"env_step": 1145000, "rew": 3422.9416, "rew_std": 196.0724, "Agent": "TD3"}, {"env_step": 1150000, "rew": 3146.4052, "rew_std": 687.3626, "Agent": "TD3"}, {"env_step": 1155000, "rew": 3223.623, "rew_std": 830.6122, "Agent": "TD3"}, {"env_step": 1160000, "rew": 3033.3122, "rew_std": 1060.267, "Agent": "TD3"}, {"env_step": 1165000, "rew": 3280.1819, "rew_std": 527.1591, "Agent": "TD3"}, {"env_step": 1170000, "rew": 3218.5815, "rew_std": 536.6197, "Agent": "TD3"}, {"env_step": 1175000, "rew": 3337.7382, "rew_std": 372.1618, "Agent": "TD3"}, {"env_step": 1180000, "rew": 3064.0044, "rew_std": 729.3349, "Agent": "TD3"}, {"env_step": 1185000, "rew": 3153.8193, "rew_std": 720.2014, "Agent": "TD3"}, {"env_step": 1190000, "rew": 2789.3759, "rew_std": 1169.6733, "Agent": "TD3"}, {"env_step": 1195000, "rew": 3351.8698, "rew_std": 493.6516, "Agent": "TD3"}, {"env_step": 1200000, "rew": 3012.5723, "rew_std": 1035.4349, "Agent": "TD3"}, {"env_step": 1205000, "rew": 3186.304, "rew_std": 730.071, "Agent": "TD3"}, {"env_step": 1210000, "rew": 3420.0329, "rew_std": 201.5622, "Agent": "TD3"}, {"env_step": 1215000, "rew": 3231.3499, "rew_std": 687.0777, "Agent": "TD3"}, {"env_step": 1220000, "rew": 3414.9551, "rew_std": 290.8079, "Agent": "TD3"}, {"env_step": 1225000, "rew": 3302.7639, "rew_std": 645.4993, "Agent": "TD3"}, {"env_step": 1230000, "rew": 3169.7956, "rew_std": 627.4685, "Agent": "TD3"}, {"env_step": 1235000, "rew": 3100.7851, "rew_std": 888.5842, "Agent": "TD3"}, {"env_step": 1240000, "rew": 3441.4178, "rew_std": 279.3749, "Agent": "TD3"}, {"env_step": 1245000, "rew": 3312.542, "rew_std": 575.5337, "Agent": "TD3"}, {"env_step": 1250000, "rew": 3097.3408, "rew_std": 746.9811, "Agent": "TD3"}, {"env_step": 0, "rew": -46.1947, "rew_std": 25.5804, "Agent": "TRPO"}, {"env_step": 30720, "rew": -13.9012, "rew_std": 11.913, "Agent": "TRPO"}, {"env_step": 61440, "rew": 18.0615, "rew_std": 9.0166, "Agent": "TRPO"}, {"env_step": 92160, "rew": 51.9175, "rew_std": 16.4955, "Agent": "TRPO"}, {"env_step": 122880, "rew": 108.4958, "rew_std": 33.2241, "Agent": "TRPO"}, {"env_step": 153600, "rew": 183.9944, "rew_std": 35.8262, "Agent": "TRPO"}, {"env_step": 184320, "rew": 257.4861, "rew_std": 60.051, "Agent": "TRPO"}, {"env_step": 215040, "rew": 430.4068, "rew_std": 124.2541, "Agent": "TRPO"}, {"env_step": 245760, "rew": 564.2017, "rew_std": 146.0606, "Agent": "TRPO"}, {"env_step": 276480, "rew": 682.5168, "rew_std": 122.3187, "Agent": "TRPO"}, {"env_step": 307200, "rew": 832.5033, "rew_std": 159.2968, "Agent": "TRPO"}, {"env_step": 337920, "rew": 1014.1049, "rew_std": 221.8108, "Agent": "TRPO"}, {"env_step": 368640, "rew": 1128.5618, "rew_std": 167.9213, "Agent": "TRPO"}, {"env_step": 399360, "rew": 1146.4494, "rew_std": 215.1602, "Agent": "TRPO"}, {"env_step": 430080, "rew": 1233.6374, "rew_std": 286.8604, "Agent": "TRPO"}, {"env_step": 460800, "rew": 1261.4876, "rew_std": 301.0892, "Agent": "TRPO"}, {"env_step": 491520, "rew": 1472.6865, "rew_std": 215.7148, "Agent": "TRPO"}, {"env_step": 522240, "rew": 1501.5343, "rew_std": 428.1053, "Agent": "TRPO"}, {"env_step": 552960, "rew": 1631.5161, "rew_std": 558.1692, "Agent": "TRPO"}, {"env_step": 583680, "rew": 1741.6005, "rew_std": 483.7599, "Agent": "TRPO"}, {"env_step": 614400, "rew": 1809.3667, "rew_std": 444.8329, "Agent": "TRPO"}, {"env_step": 645120, "rew": 1943.2351, "rew_std": 386.0455, "Agent": "TRPO"}, {"env_step": 675840, "rew": 2119.3593, "rew_std": 463.9853, "Agent": "TRPO"}, {"env_step": 706560, "rew": 2160.9349, "rew_std": 299.8534, "Agent": "TRPO"}, {"env_step": 737280, "rew": 2205.6584, "rew_std": 495.8226, "Agent": "TRPO"}, {"env_step": 768000, "rew": 2148.4616, "rew_std": 617.4263, "Agent": "TRPO"}, {"env_step": 798720, "rew": 2294.2347, "rew_std": 619.654, "Agent": "TRPO"}, {"env_step": 829440, "rew": 2334.3877, "rew_std": 569.8366, "Agent": "TRPO"}, {"env_step": 860160, "rew": 2373.0195, "rew_std": 635.4872, "Agent": "TRPO"}, {"env_step": 890880, "rew": 2572.1874, "rew_std": 771.2781, "Agent": "TRPO"}, {"env_step": 921600, "rew": 2568.2581, "rew_std": 539.7726, "Agent": "TRPO"}, {"env_step": 952320, "rew": 2639.2867, "rew_std": 572.4301, "Agent": "TRPO"}, {"env_step": 983040, "rew": 2866.6508, "rew_std": 707.8937, "Agent": "TRPO"}, {"env_step": 1013760, "rew": 2731.0697, "rew_std": 675.644, "Agent": "TRPO"}, {"env_step": 1044480, "rew": 2702.1965, "rew_std": 742.4807, "Agent": "TRPO"}, {"env_step": 1075200, "rew": 3001.7621, "rew_std": 688.8314, "Agent": "TRPO"}, {"env_step": 1105920, "rew": 2877.8716, "rew_std": 922.2411, "Agent": "TRPO"}, {"env_step": 1136640, "rew": 3104.9072, "rew_std": 800.8227, "Agent": "TRPO"}, {"env_step": 1167360, "rew": 2976.5528, "rew_std": 661.5153, "Agent": "TRPO"}, {"env_step": 1198080, "rew": 3196.2228, "rew_std": 707.9679, "Agent": "TRPO"}, {"env_step": 1228800, "rew": 3237.2005, "rew_std": 505.6685, "Agent": "TRPO"}, {"env_step": 1259520, "rew": 3098.3083, "rew_std": 578.4346, "Agent": "TRPO"}, {"env_step": 1290240, "rew": 3356.962, "rew_std": 786.3838, "Agent": "TRPO"}, {"env_step": 1320960, "rew": 3241.2647, "rew_std": 600.3495, "Agent": "TRPO"}, {"env_step": 1351680, "rew": 3361.1722, "rew_std": 547.5151, "Agent": "TRPO"}, {"env_step": 1382400, "rew": 3495.3879, "rew_std": 662.5387, "Agent": "TRPO"}, {"env_step": 1413120, "rew": 3388.8443, "rew_std": 537.1016, "Agent": "TRPO"}, {"env_step": 1443840, "rew": 3493.6456, "rew_std": 509.2391, "Agent": "TRPO"}, {"env_step": 1474560, "rew": 3256.3625, "rew_std": 412.1903, "Agent": "TRPO"}, {"env_step": 1505280, "rew": 3695.3862, "rew_std": 644.5284, "Agent": "TRPO"}, {"env_step": 1536000, "rew": 3685.0253, "rew_std": 537.4812, "Agent": "TRPO"}, {"env_step": 1566720, "rew": 3565.3736, "rew_std": 647.2313, "Agent": "TRPO"}, {"env_step": 1597440, "rew": 3790.5153, "rew_std": 702.1943, "Agent": "TRPO"}, {"env_step": 1628160, "rew": 3598.3267, "rew_std": 626.2541, "Agent": "TRPO"}, {"env_step": 1658880, "rew": 3970.7287, "rew_std": 647.9836, "Agent": "TRPO"}, {"env_step": 1689600, "rew": 3658.6492, "rew_std": 645.0697, "Agent": "TRPO"}, {"env_step": 1720320, "rew": 3848.0345, "rew_std": 701.1273, "Agent": "TRPO"}, {"env_step": 1751040, "rew": 4061.4513, "rew_std": 690.0289, "Agent": "TRPO"}, {"env_step": 1781760, "rew": 3807.921, "rew_std": 706.5999, "Agent": "TRPO"}, {"env_step": 1812480, "rew": 3802.0342, "rew_std": 663.7139, "Agent": "TRPO"}, {"env_step": 1843200, "rew": 3873.6662, "rew_std": 566.3833, "Agent": "TRPO"}, {"env_step": 1873920, "rew": 4038.6082, "rew_std": 518.5079, "Agent": "TRPO"}, {"env_step": 1904640, "rew": 4167.3221, "rew_std": 440.9658, "Agent": "TRPO"}, {"env_step": 1935360, "rew": 3922.2855, "rew_std": 733.8238, "Agent": "TRPO"}, {"env_step": 1966080, "rew": 4174.7307, "rew_std": 495.1849, "Agent": "TRPO"}, {"env_step": 1996800, "rew": 4155.8384, "rew_std": 477.6376, "Agent": "TRPO"}, {"env_step": 2027520, "rew": 4259.3055, "rew_std": 577.3636, "Agent": "TRPO"}, {"env_step": 2058240, "rew": 4256.1933, "rew_std": 594.2826, "Agent": "TRPO"}, {"env_step": 2088960, "rew": 4355.0669, "rew_std": 605.8978, "Agent": "TRPO"}, {"env_step": 2119680, "rew": 4297.2252, "rew_std": 474.421, "Agent": "TRPO"}, {"env_step": 2150400, "rew": 4287.4698, "rew_std": 363.7332, "Agent": "TRPO"}, {"env_step": 2181120, "rew": 4398.2137, "rew_std": 354.3603, "Agent": "TRPO"}, {"env_step": 2211840, "rew": 4413.5796, "rew_std": 460.9681, "Agent": "TRPO"}, {"env_step": 2242560, "rew": 4329.001, "rew_std": 530.2755, "Agent": "TRPO"}, {"env_step": 2273280, "rew": 4252.3464, "rew_std": 597.5882, "Agent": "TRPO"}, {"env_step": 2304000, "rew": 4462.5657, "rew_std": 495.3802, "Agent": "TRPO"}, {"env_step": 2334720, "rew": 4374.2838, "rew_std": 457.0956, "Agent": "TRPO"}, {"env_step": 2365440, "rew": 4529.4549, "rew_std": 461.1733, "Agent": "TRPO"}, {"env_step": 2396160, "rew": 4291.4468, "rew_std": 411.2142, "Agent": "TRPO"}, {"env_step": 2426880, "rew": 4430.1459, "rew_std": 601.8812, "Agent": "TRPO"}, {"env_step": 2457600, "rew": 4508.078, "rew_std": 465.0507, "Agent": "TRPO"}, {"env_step": 2488320, "rew": 4151.3796, "rew_std": 414.656, "Agent": "TRPO"}, {"env_step": 2519040, "rew": 4705.7212, "rew_std": 477.1206, "Agent": "TRPO"}, {"env_step": 2549760, "rew": 4641.3111, "rew_std": 622.6369, "Agent": "TRPO"}, {"env_step": 2580480, "rew": 4484.1452, "rew_std": 442.0811, "Agent": "TRPO"}, {"env_step": 2611200, "rew": 4658.7294, "rew_std": 432.1496, "Agent": "TRPO"}, {"env_step": 2641920, "rew": 4625.8785, "rew_std": 357.4208, "Agent": "TRPO"}, {"env_step": 2672640, "rew": 4284.9794, "rew_std": 599.7262, "Agent": "TRPO"}, {"env_step": 2703360, "rew": 4419.1972, "rew_std": 447.5325, "Agent": "TRPO"}, {"env_step": 2734080, "rew": 4746.1067, "rew_std": 580.4515, "Agent": "TRPO"}, {"env_step": 2764800, "rew": 4756.6644, "rew_std": 502.4658, "Agent": "TRPO"}, {"env_step": 2795520, "rew": 4607.7287, "rew_std": 427.1984, "Agent": "TRPO"}, {"env_step": 2826240, "rew": 4404.4363, "rew_std": 555.3297, "Agent": "TRPO"}, {"env_step": 2856960, "rew": 4816.8623, "rew_std": 528.6794, "Agent": "TRPO"}, {"env_step": 2887680, "rew": 4753.3578, "rew_std": 715.7057, "Agent": "TRPO"}, {"env_step": 2918400, "rew": 4604.763, "rew_std": 447.4458, "Agent": "TRPO"}, {"env_step": 2949120, "rew": 4800.2693, "rew_std": 602.9667, "Agent": "TRPO"}, {"env_step": 2979840, "rew": 4772.9396, "rew_std": 368.1691, "Agent": "TRPO"}, {"env_step": 3010560, "rew": 4897.0257, "rew_std": 449.1329, "Agent": "TRPO"}, {"env_step": 3041280, "rew": 4841.7814, "rew_std": 534.8951, "Agent": "TRPO"}, {"env_step": 3072000, "rew": 4604.9043, "rew_std": 580.976, "Agent": "TRPO"}, {"env_step": 0, "rew": -46.1947, "rew_std": 25.5804, "Agent": "NPG"}, {"env_step": 30720, "rew": -29.0009, "rew_std": 15.8916, "Agent": "NPG"}, {"env_step": 61440, "rew": -10.4279, "rew_std": 11.0242, "Agent": "NPG"}, {"env_step": 92160, "rew": 11.1187, "rew_std": 9.0737, "Agent": "NPG"}, {"env_step": 122880, "rew": 27.5852, "rew_std": 16.4638, "Agent": "NPG"}, {"env_step": 153600, "rew": 56.7285, "rew_std": 23.8642, "Agent": "NPG"}, {"env_step": 184320, "rew": 81.2319, "rew_std": 29.1525, "Agent": "NPG"}, {"env_step": 215040, "rew": 159.4704, "rew_std": 47.3793, "Agent": "NPG"}, {"env_step": 245760, "rew": 174.9781, "rew_std": 57.693, "Agent": "NPG"}, {"env_step": 276480, "rew": 178.6358, "rew_std": 32.807, "Agent": "NPG"}, {"env_step": 307200, "rew": 218.3212, "rew_std": 41.035, "Agent": "NPG"}, {"env_step": 337920, "rew": 368.663, "rew_std": 80.0623, "Agent": "NPG"}, {"env_step": 368640, "rew": 432.2163, "rew_std": 63.1225, "Agent": "NPG"}, {"env_step": 399360, "rew": 568.0807, "rew_std": 125.517, "Agent": "NPG"}, {"env_step": 430080, "rew": 631.1259, "rew_std": 114.6371, "Agent": "NPG"}, {"env_step": 460800, "rew": 753.5929, "rew_std": 118.5185, "Agent": "NPG"}, {"env_step": 491520, "rew": 864.7371, "rew_std": 183.4071, "Agent": "NPG"}, {"env_step": 522240, "rew": 940.2978, "rew_std": 158.8018, "Agent": "NPG"}, {"env_step": 552960, "rew": 1021.7661, "rew_std": 159.2406, "Agent": "NPG"}, {"env_step": 583680, "rew": 1131.714, "rew_std": 150.4756, "Agent": "NPG"}, {"env_step": 614400, "rew": 1201.5181, "rew_std": 164.258, "Agent": "NPG"}, {"env_step": 645120, "rew": 1343.1191, "rew_std": 184.6686, "Agent": "NPG"}, {"env_step": 675840, "rew": 1318.401, "rew_std": 229.7468, "Agent": "NPG"}, {"env_step": 706560, "rew": 1459.1159, "rew_std": 222.6298, "Agent": "NPG"}, {"env_step": 737280, "rew": 1542.0295, "rew_std": 203.9479, "Agent": "NPG"}, {"env_step": 768000, "rew": 1634.9176, "rew_std": 218.2131, "Agent": "NPG"}, {"env_step": 798720, "rew": 1692.0983, "rew_std": 343.9785, "Agent": "NPG"}, {"env_step": 829440, "rew": 1906.1507, "rew_std": 457.5102, "Agent": "NPG"}, {"env_step": 860160, "rew": 1775.1748, "rew_std": 342.0144, "Agent": "NPG"}, {"env_step": 890880, "rew": 2102.0494, "rew_std": 355.8503, "Agent": "NPG"}, {"env_step": 921600, "rew": 2046.7794, "rew_std": 340.8884, "Agent": "NPG"}, {"env_step": 952320, "rew": 2164.3512, "rew_std": 405.7482, "Agent": "NPG"}, {"env_step": 983040, "rew": 2348.312, "rew_std": 509.9401, "Agent": "NPG"}, {"env_step": 1013760, "rew": 2365.8958, "rew_std": 523.6603, "Agent": "NPG"}, {"env_step": 1044480, "rew": 2324.9164, "rew_std": 528.4526, "Agent": "NPG"}, {"env_step": 1075200, "rew": 2493.722, "rew_std": 583.8895, "Agent": "NPG"}, {"env_step": 1105920, "rew": 2638.7282, "rew_std": 650.3356, "Agent": "NPG"}, {"env_step": 1136640, "rew": 2588.3598, "rew_std": 506.4239, "Agent": "NPG"}, {"env_step": 1167360, "rew": 2657.8218, "rew_std": 542.1361, "Agent": "NPG"}, {"env_step": 1198080, "rew": 2772.0887, "rew_std": 399.8722, "Agent": "NPG"}, {"env_step": 1228800, "rew": 2983.8532, "rew_std": 405.1427, "Agent": "NPG"}, {"env_step": 1259520, "rew": 2969.912, "rew_std": 543.0224, "Agent": "NPG"}, {"env_step": 1290240, "rew": 2898.5856, "rew_std": 687.1927, "Agent": "NPG"}, {"env_step": 1320960, "rew": 2979.687, "rew_std": 685.3181, "Agent": "NPG"}, {"env_step": 1351680, "rew": 2937.7865, "rew_std": 654.2811, "Agent": "NPG"}, {"env_step": 1382400, "rew": 3180.3262, "rew_std": 502.1675, "Agent": "NPG"}, {"env_step": 1413120, "rew": 3181.4746, "rew_std": 526.9918, "Agent": "NPG"}, {"env_step": 1443840, "rew": 3186.725, "rew_std": 483.0104, "Agent": "NPG"}, {"env_step": 1474560, "rew": 3347.1624, "rew_std": 474.3212, "Agent": "NPG"}, {"env_step": 1505280, "rew": 3264.1247, "rew_std": 542.1326, "Agent": "NPG"}, {"env_step": 1536000, "rew": 3267.3737, "rew_std": 536.477, "Agent": "NPG"}, {"env_step": 1566720, "rew": 3677.709, "rew_std": 486.0857, "Agent": "NPG"}, {"env_step": 1597440, "rew": 3379.1083, "rew_std": 683.1763, "Agent": "NPG"}, {"env_step": 1628160, "rew": 3533.6653, "rew_std": 437.9623, "Agent": "NPG"}, {"env_step": 1658880, "rew": 3534.0292, "rew_std": 787.6867, "Agent": "NPG"}, {"env_step": 1689600, "rew": 3399.7589, "rew_std": 685.1419, "Agent": "NPG"}, {"env_step": 1720320, "rew": 3587.7024, "rew_std": 471.134, "Agent": "NPG"}, {"env_step": 1751040, "rew": 3560.6233, "rew_std": 616.8093, "Agent": "NPG"}, {"env_step": 1781760, "rew": 3635.7303, "rew_std": 608.4666, "Agent": "NPG"}, {"env_step": 1812480, "rew": 3713.5899, "rew_std": 802.8577, "Agent": "NPG"}, {"env_step": 1843200, "rew": 3590.908, "rew_std": 557.7779, "Agent": "NPG"}, {"env_step": 1873920, "rew": 3903.7812, "rew_std": 655.7805, "Agent": "NPG"}, {"env_step": 1904640, "rew": 3770.5075, "rew_std": 645.9229, "Agent": "NPG"}, {"env_step": 1935360, "rew": 4110.3599, "rew_std": 657.3085, "Agent": "NPG"}, {"env_step": 1966080, "rew": 3851.8914, "rew_std": 745.2079, "Agent": "NPG"}, {"env_step": 1996800, "rew": 3998.0014, "rew_std": 650.7886, "Agent": "NPG"}, {"env_step": 2027520, "rew": 3739.3657, "rew_std": 715.6727, "Agent": "NPG"}, {"env_step": 2058240, "rew": 4230.5764, "rew_std": 578.5901, "Agent": "NPG"}, {"env_step": 2088960, "rew": 4140.935, "rew_std": 624.3245, "Agent": "NPG"}, {"env_step": 2119680, "rew": 4228.9808, "rew_std": 573.9637, "Agent": "NPG"}, {"env_step": 2150400, "rew": 4142.6649, "rew_std": 419.5781, "Agent": "NPG"}, {"env_step": 2181120, "rew": 4089.8147, "rew_std": 302.9776, "Agent": "NPG"}, {"env_step": 2211840, "rew": 4276.8854, "rew_std": 674.4424, "Agent": "NPG"}, {"env_step": 2242560, "rew": 4312.1654, "rew_std": 567.5481, "Agent": "NPG"}, {"env_step": 2273280, "rew": 4224.3566, "rew_std": 445.0954, "Agent": "NPG"}, {"env_step": 2304000, "rew": 3878.7632, "rew_std": 866.3551, "Agent": "NPG"}, {"env_step": 2334720, "rew": 4001.3801, "rew_std": 478.1705, "Agent": "NPG"}, {"env_step": 2365440, "rew": 4284.5594, "rew_std": 448.8121, "Agent": "NPG"}, {"env_step": 2396160, "rew": 4406.6923, "rew_std": 516.8683, "Agent": "NPG"}, {"env_step": 2426880, "rew": 4145.9032, "rew_std": 519.4268, "Agent": "NPG"}, {"env_step": 2457600, "rew": 4241.7632, "rew_std": 685.6791, "Agent": "NPG"}, {"env_step": 2488320, "rew": 4154.1219, "rew_std": 437.1567, "Agent": "NPG"}, {"env_step": 2519040, "rew": 4495.7374, "rew_std": 449.1806, "Agent": "NPG"}, {"env_step": 2549760, "rew": 4289.7487, "rew_std": 526.7084, "Agent": "NPG"}, {"env_step": 2580480, "rew": 4246.0211, "rew_std": 530.6265, "Agent": "NPG"}, {"env_step": 2611200, "rew": 4463.8985, "rew_std": 572.8797, "Agent": "NPG"}, {"env_step": 2641920, "rew": 4298.6282, "rew_std": 665.9416, "Agent": "NPG"}, {"env_step": 2672640, "rew": 4485.5394, "rew_std": 359.9158, "Agent": "NPG"}, {"env_step": 2703360, "rew": 4820.597, "rew_std": 318.5627, "Agent": "NPG"}, {"env_step": 2734080, "rew": 4540.1727, "rew_std": 412.9545, "Agent": "NPG"}, {"env_step": 2764800, "rew": 4500.1939, "rew_std": 596.1343, "Agent": "NPG"}, {"env_step": 2795520, "rew": 4664.0692, "rew_std": 426.6286, "Agent": "NPG"}, {"env_step": 2826240, "rew": 4516.467, "rew_std": 435.656, "Agent": "NPG"}, {"env_step": 2856960, "rew": 4545.139, "rew_std": 496.411, "Agent": "NPG"}, {"env_step": 2887680, "rew": 4444.0518, "rew_std": 596.6213, "Agent": "NPG"}, {"env_step": 2918400, "rew": 4658.8307, "rew_std": 652.0561, "Agent": "NPG"}, {"env_step": 2949120, "rew": 4509.4439, "rew_std": 466.3483, "Agent": "NPG"}, {"env_step": 2979840, "rew": 4589.8375, "rew_std": 414.0363, "Agent": "NPG"}, {"env_step": 3010560, "rew": 4693.0902, "rew_std": 490.4674, "Agent": "NPG"}, {"env_step": 3041280, "rew": 4678.8152, "rew_std": 313.0054, "Agent": "NPG"}, {"env_step": 3072000, "rew": 4736.6273, "rew_std": 376.7205, "Agent": "NPG"}, {"env_step": 0, "rew": 17.4591, "rew_std": 3.1393, "Agent": "PPO"}, {"env_step": 30720, "rew": 258.0573, "rew_std": 21.2869, "Agent": "PPO"}, {"env_step": 61440, "rew": 321.5229, "rew_std": 32.9819, "Agent": "PPO"}, {"env_step": 92160, "rew": 368.9201, "rew_std": 45.9052, "Agent": "PPO"}, {"env_step": 122880, "rew": 399.3288, "rew_std": 43.0806, "Agent": "PPO"}, {"env_step": 153600, "rew": 432.5139, "rew_std": 71.3239, "Agent": "PPO"}, {"env_step": 184320, "rew": 499.772, "rew_std": 111.9788, "Agent": "PPO"}, {"env_step": 215040, "rew": 555.8803, "rew_std": 145.4804, "Agent": "PPO"}, {"env_step": 245760, "rew": 676.9764, "rew_std": 211.6327, "Agent": "PPO"}, {"env_step": 276480, "rew": 813.3559, "rew_std": 313.94, "Agent": "PPO"}, {"env_step": 307200, "rew": 883.8523, "rew_std": 297.6962, "Agent": "PPO"}, {"env_step": 337920, "rew": 965.7387, "rew_std": 317.0964, "Agent": "PPO"}, {"env_step": 368640, "rew": 1133.3987, "rew_std": 313.3517, "Agent": "PPO"}, {"env_step": 399360, "rew": 1301.2343, "rew_std": 435.8154, "Agent": "PPO"}, {"env_step": 430080, "rew": 1508.3458, "rew_std": 468.8199, "Agent": "PPO"}, {"env_step": 460800, "rew": 2025.0004, "rew_std": 756.4872, "Agent": "PPO"}, {"env_step": 491520, "rew": 1975.8486, "rew_std": 705.2052, "Agent": "PPO"}, {"env_step": 522240, "rew": 2227.2255, "rew_std": 669.5327, "Agent": "PPO"}, {"env_step": 552960, "rew": 2409.7045, "rew_std": 716.1465, "Agent": "PPO"}, {"env_step": 583680, "rew": 2393.2191, "rew_std": 703.3068, "Agent": "PPO"}, {"env_step": 614400, "rew": 2312.3677, "rew_std": 731.6068, "Agent": "PPO"}, {"env_step": 645120, "rew": 2165.5728, "rew_std": 703.9315, "Agent": "PPO"}, {"env_step": 675840, "rew": 2199.1089, "rew_std": 663.8891, "Agent": "PPO"}, {"env_step": 706560, "rew": 2253.354, "rew_std": 723.8256, "Agent": "PPO"}, {"env_step": 737280, "rew": 2047.2415, "rew_std": 670.5035, "Agent": "PPO"}, {"env_step": 768000, "rew": 2268.8839, "rew_std": 920.2149, "Agent": "PPO"}, {"env_step": 798720, "rew": 2331.4466, "rew_std": 660.495, "Agent": "PPO"}, {"env_step": 829440, "rew": 2395.9565, "rew_std": 766.3797, "Agent": "PPO"}, {"env_step": 860160, "rew": 2412.8335, "rew_std": 845.1709, "Agent": "PPO"}, {"env_step": 890880, "rew": 2609.335, "rew_std": 700.8103, "Agent": "PPO"}, {"env_step": 921600, "rew": 2323.7125, "rew_std": 554.5084, "Agent": "PPO"}, {"env_step": 952320, "rew": 2342.3823, "rew_std": 677.2163, "Agent": "PPO"}, {"env_step": 983040, "rew": 2370.1043, "rew_std": 695.5023, "Agent": "PPO"}, {"env_step": 1013760, "rew": 2482.5501, "rew_std": 852.3902, "Agent": "PPO"}, {"env_step": 1044480, "rew": 2543.0648, "rew_std": 664.076, "Agent": "PPO"}, {"env_step": 1075200, "rew": 2595.0248, "rew_std": 682.0517, "Agent": "PPO"}, {"env_step": 1105920, "rew": 2293.5028, "rew_std": 619.1544, "Agent": "PPO"}, {"env_step": 1136640, "rew": 2665.0402, "rew_std": 689.8843, "Agent": "PPO"}, {"env_step": 1167360, "rew": 2365.2063, "rew_std": 640.2129, "Agent": "PPO"}, {"env_step": 1198080, "rew": 2740.7154, "rew_std": 743.8731, "Agent": "PPO"}, {"env_step": 1228800, "rew": 2375.3191, "rew_std": 919.2542, "Agent": "PPO"}, {"env_step": 1259520, "rew": 2752.1238, "rew_std": 635.7227, "Agent": "PPO"}, {"env_step": 1290240, "rew": 2652.1802, "rew_std": 545.9033, "Agent": "PPO"}, {"env_step": 1320960, "rew": 2563.4924, "rew_std": 543.7153, "Agent": "PPO"}, {"env_step": 1351680, "rew": 2903.989, "rew_std": 437.1591, "Agent": "PPO"}, {"env_step": 1382400, "rew": 2708.8197, "rew_std": 442.3195, "Agent": "PPO"}, {"env_step": 1413120, "rew": 2696.5147, "rew_std": 533.7476, "Agent": "PPO"}, {"env_step": 1443840, "rew": 2825.6167, "rew_std": 502.3265, "Agent": "PPO"}, {"env_step": 1474560, "rew": 2837.7101, "rew_std": 583.6296, "Agent": "PPO"}, {"env_step": 1505280, "rew": 2286.467, "rew_std": 775.9048, "Agent": "PPO"}, {"env_step": 1536000, "rew": 2564.4424, "rew_std": 609.329, "Agent": "PPO"}, {"env_step": 1566720, "rew": 2815.8548, "rew_std": 473.1315, "Agent": "PPO"}, {"env_step": 1597440, "rew": 2637.735, "rew_std": 407.1216, "Agent": "PPO"}, {"env_step": 1628160, "rew": 2826.9888, "rew_std": 466.1343, "Agent": "PPO"}, {"env_step": 1658880, "rew": 2679.819, "rew_std": 604.9165, "Agent": "PPO"}, {"env_step": 1689600, "rew": 2310.0095, "rew_std": 820.3102, "Agent": "PPO"}, {"env_step": 1720320, "rew": 2401.5418, "rew_std": 607.9464, "Agent": "PPO"}, {"env_step": 1751040, "rew": 2253.5243, "rew_std": 640.3227, "Agent": "PPO"}, {"env_step": 1781760, "rew": 2329.541, "rew_std": 569.5139, "Agent": "PPO"}, {"env_step": 1812480, "rew": 2402.1102, "rew_std": 546.2053, "Agent": "PPO"}, {"env_step": 1843200, "rew": 2731.4867, "rew_std": 377.6566, "Agent": "PPO"}, {"env_step": 1873920, "rew": 2638.8958, "rew_std": 423.2264, "Agent": "PPO"}, {"env_step": 1904640, "rew": 2741.9472, "rew_std": 490.6814, "Agent": "PPO"}, {"env_step": 1935360, "rew": 2672.56, "rew_std": 626.8115, "Agent": "PPO"}, {"env_step": 1966080, "rew": 2579.2487, "rew_std": 823.8814, "Agent": "PPO"}, {"env_step": 1996800, "rew": 2532.4619, "rew_std": 608.9239, "Agent": "PPO"}, {"env_step": 2027520, "rew": 2714.8373, "rew_std": 558.9923, "Agent": "PPO"}, {"env_step": 2058240, "rew": 2790.1972, "rew_std": 606.7406, "Agent": "PPO"}, {"env_step": 2088960, "rew": 2806.6684, "rew_std": 594.6761, "Agent": "PPO"}, {"env_step": 2119680, "rew": 2983.2358, "rew_std": 490.9231, "Agent": "PPO"}, {"env_step": 2150400, "rew": 2678.8518, "rew_std": 383.2641, "Agent": "PPO"}, {"env_step": 2181120, "rew": 2504.8482, "rew_std": 731.2124, "Agent": "PPO"}, {"env_step": 2211840, "rew": 2916.3325, "rew_std": 529.2655, "Agent": "PPO"}, {"env_step": 2242560, "rew": 2736.5338, "rew_std": 549.6671, "Agent": "PPO"}, {"env_step": 2273280, "rew": 2704.9419, "rew_std": 624.3988, "Agent": "PPO"}, {"env_step": 2304000, "rew": 2578.0559, "rew_std": 619.9542, "Agent": "PPO"}, {"env_step": 2334720, "rew": 2459.1878, "rew_std": 714.9442, "Agent": "PPO"}, {"env_step": 2365440, "rew": 2359.8527, "rew_std": 534.6146, "Agent": "PPO"}, {"env_step": 2396160, "rew": 2530.1409, "rew_std": 724.2738, "Agent": "PPO"}, {"env_step": 2426880, "rew": 2624.1701, "rew_std": 535.1847, "Agent": "PPO"}, {"env_step": 2457600, "rew": 2731.1249, "rew_std": 553.7215, "Agent": "PPO"}, {"env_step": 2488320, "rew": 2601.1055, "rew_std": 374.4485, "Agent": "PPO"}, {"env_step": 2519040, "rew": 2680.2664, "rew_std": 439.8017, "Agent": "PPO"}, {"env_step": 2549760, "rew": 2991.4033, "rew_std": 377.8164, "Agent": "PPO"}, {"env_step": 2580480, "rew": 2935.6051, "rew_std": 455.1756, "Agent": "PPO"}, {"env_step": 2611200, "rew": 3086.7075, "rew_std": 346.5505, "Agent": "PPO"}, {"env_step": 2641920, "rew": 2894.2612, "rew_std": 316.5737, "Agent": "PPO"}, {"env_step": 2672640, "rew": 3127.6851, "rew_std": 413.0159, "Agent": "PPO"}, {"env_step": 2703360, "rew": 2947.9566, "rew_std": 440.9114, "Agent": "PPO"}, {"env_step": 2734080, "rew": 3080.4229, "rew_std": 392.3963, "Agent": "PPO"}, {"env_step": 2764800, "rew": 2959.3085, "rew_std": 509.0458, "Agent": "PPO"}, {"env_step": 2795520, "rew": 2998.2617, "rew_std": 363.2342, "Agent": "PPO"}, {"env_step": 2826240, "rew": 3025.9286, "rew_std": 327.0367, "Agent": "PPO"}, {"env_step": 2856960, "rew": 2685.6607, "rew_std": 479.2221, "Agent": "PPO"}, {"env_step": 2887680, "rew": 2904.1953, "rew_std": 525.6346, "Agent": "PPO"}, {"env_step": 2918400, "rew": 2766.6171, "rew_std": 547.6156, "Agent": "PPO"}, {"env_step": 2949120, "rew": 2775.0355, "rew_std": 490.8935, "Agent": "PPO"}, {"env_step": 2979840, "rew": 2534.7584, "rew_std": 647.3288, "Agent": "PPO"}, {"env_step": 3010560, "rew": 2747.8063, "rew_std": 647.902, "Agent": "PPO"}, {"env_step": 3041280, "rew": 2786.2628, "rew_std": 657.6806, "Agent": "PPO"}, {"env_step": 3072000, "rew": 2714.6257, "rew_std": 651.8296, "Agent": "PPO"}, {"env_step": 0, "rew": 17.9154, "rew_std": 1.9743, "Agent": "REINFORCE"}, {"env_step": 30720, "rew": 29.3731, "rew_std": 10.4893, "Agent": "REINFORCE"}, {"env_step": 61440, "rew": 66.6037, "rew_std": 24.3997, "Agent": "REINFORCE"}, {"env_step": 92160, "rew": 198.8276, "rew_std": 36.86, "Agent": "REINFORCE"}, {"env_step": 122880, "rew": 232.7992, "rew_std": 26.5837, "Agent": "REINFORCE"}, {"env_step": 153600, "rew": 260.3133, "rew_std": 22.0087, "Agent": "REINFORCE"}, {"env_step": 184320, "rew": 268.9626, "rew_std": 28.6902, "Agent": "REINFORCE"}, {"env_step": 215040, "rew": 288.7649, "rew_std": 32.1015, "Agent": "REINFORCE"}, {"env_step": 245760, "rew": 295.2648, "rew_std": 22.7345, "Agent": "REINFORCE"}, {"env_step": 276480, "rew": 306.5046, "rew_std": 15.8915, "Agent": "REINFORCE"}, {"env_step": 307200, "rew": 318.4252, "rew_std": 9.9478, "Agent": "REINFORCE"}, {"env_step": 337920, "rew": 322.6908, "rew_std": 17.5204, "Agent": "REINFORCE"}, {"env_step": 368640, "rew": 336.2168, "rew_std": 28.7181, "Agent": "REINFORCE"}, {"env_step": 399360, "rew": 330.8605, "rew_std": 45.2815, "Agent": "REINFORCE"}, {"env_step": 430080, "rew": 331.2807, "rew_std": 23.8464, "Agent": "REINFORCE"}, {"env_step": 460800, "rew": 350.7925, "rew_std": 64.5441, "Agent": "REINFORCE"}, {"env_step": 491520, "rew": 341.1786, "rew_std": 43.6435, "Agent": "REINFORCE"}, {"env_step": 522240, "rew": 346.7587, "rew_std": 38.0414, "Agent": "REINFORCE"}, {"env_step": 552960, "rew": 343.9709, "rew_std": 40.9004, "Agent": "REINFORCE"}, {"env_step": 583680, "rew": 346.0514, "rew_std": 47.826, "Agent": "REINFORCE"}, {"env_step": 614400, "rew": 355.0738, "rew_std": 51.1227, "Agent": "REINFORCE"}, {"env_step": 645120, "rew": 353.4741, "rew_std": 39.3418, "Agent": "REINFORCE"}, {"env_step": 675840, "rew": 363.9368, "rew_std": 53.8079, "Agent": "REINFORCE"}, {"env_step": 706560, "rew": 369.7521, "rew_std": 76.4353, "Agent": "REINFORCE"}, {"env_step": 737280, "rew": 362.2687, "rew_std": 94.7053, "Agent": "REINFORCE"}, {"env_step": 768000, "rew": 367.0035, "rew_std": 79.9643, "Agent": "REINFORCE"}, {"env_step": 798720, "rew": 365.8843, "rew_std": 73.7307, "Agent": "REINFORCE"}, {"env_step": 829440, "rew": 370.4545, "rew_std": 74.0224, "Agent": "REINFORCE"}, {"env_step": 860160, "rew": 354.3553, "rew_std": 59.5886, "Agent": "REINFORCE"}, {"env_step": 890880, "rew": 386.9329, "rew_std": 76.6984, "Agent": "REINFORCE"}, {"env_step": 921600, "rew": 386.7397, "rew_std": 80.6028, "Agent": "REINFORCE"}, {"env_step": 952320, "rew": 387.5511, "rew_std": 95.2815, "Agent": "REINFORCE"}, {"env_step": 983040, "rew": 389.9666, "rew_std": 84.4154, "Agent": "REINFORCE"}, {"env_step": 1013760, "rew": 374.6943, "rew_std": 68.1604, "Agent": "REINFORCE"}, {"env_step": 1044480, "rew": 387.5142, "rew_std": 68.4897, "Agent": "REINFORCE"}, {"env_step": 1075200, "rew": 379.5257, "rew_std": 61.456, "Agent": "REINFORCE"}, {"env_step": 1105920, "rew": 382.3943, "rew_std": 63.4778, "Agent": "REINFORCE"}, {"env_step": 1136640, "rew": 376.9915, "rew_std": 69.3571, "Agent": "REINFORCE"}, {"env_step": 1167360, "rew": 386.4603, "rew_std": 67.3477, "Agent": "REINFORCE"}, {"env_step": 1198080, "rew": 393.3204, "rew_std": 79.687, "Agent": "REINFORCE"}, {"env_step": 1228800, "rew": 375.8716, "rew_std": 49.7071, "Agent": "REINFORCE"}, {"env_step": 1259520, "rew": 377.5592, "rew_std": 60.5279, "Agent": "REINFORCE"}, {"env_step": 1290240, "rew": 369.21, "rew_std": 29.6277, "Agent": "REINFORCE"}, {"env_step": 1320960, "rew": 373.6731, "rew_std": 42.2747, "Agent": "REINFORCE"}, {"env_step": 1351680, "rew": 363.1084, "rew_std": 27.0787, "Agent": "REINFORCE"}, {"env_step": 1382400, "rew": 374.9453, "rew_std": 43.472, "Agent": "REINFORCE"}, {"env_step": 1413120, "rew": 372.8838, "rew_std": 46.91, "Agent": "REINFORCE"}, {"env_step": 1443840, "rew": 382.9686, "rew_std": 51.2128, "Agent": "REINFORCE"}, {"env_step": 1474560, "rew": 374.1388, "rew_std": 55.2316, "Agent": "REINFORCE"}, {"env_step": 1505280, "rew": 370.8528, "rew_std": 62.1252, "Agent": "REINFORCE"}, {"env_step": 1536000, "rew": 372.8432, "rew_std": 62.8749, "Agent": "REINFORCE"}, {"env_step": 1566720, "rew": 381.4441, "rew_std": 67.9162, "Agent": "REINFORCE"}, {"env_step": 1597440, "rew": 393.132, "rew_std": 66.7161, "Agent": "REINFORCE"}, {"env_step": 1628160, "rew": 387.3093, "rew_std": 79.775, "Agent": "REINFORCE"}, {"env_step": 1658880, "rew": 391.0778, "rew_std": 64.053, "Agent": "REINFORCE"}, {"env_step": 1689600, "rew": 395.8294, "rew_std": 64.4886, "Agent": "REINFORCE"}, {"env_step": 1720320, "rew": 392.6451, "rew_std": 68.072, "Agent": "REINFORCE"}, {"env_step": 1751040, "rew": 390.7222, "rew_std": 58.8086, "Agent": "REINFORCE"}, {"env_step": 1781760, "rew": 374.8847, "rew_std": 39.4855, "Agent": "REINFORCE"}, {"env_step": 1812480, "rew": 361.9808, "rew_std": 49.6531, "Agent": "REINFORCE"}, {"env_step": 1843200, "rew": 386.8282, "rew_std": 66.9517, "Agent": "REINFORCE"}, {"env_step": 1873920, "rew": 373.0323, "rew_std": 63.2295, "Agent": "REINFORCE"}, {"env_step": 1904640, "rew": 373.7938, "rew_std": 69.8201, "Agent": "REINFORCE"}, {"env_step": 1935360, "rew": 376.2475, "rew_std": 77.6855, "Agent": "REINFORCE"}, {"env_step": 1966080, "rew": 380.674, "rew_std": 81.9143, "Agent": "REINFORCE"}, {"env_step": 1996800, "rew": 377.8038, "rew_std": 70.1288, "Agent": "REINFORCE"}, {"env_step": 2027520, "rew": 368.3728, "rew_std": 80.0865, "Agent": "REINFORCE"}, {"env_step": 2058240, "rew": 363.0808, "rew_std": 67.5735, "Agent": "REINFORCE"}, {"env_step": 2088960, "rew": 371.7142, "rew_std": 60.2024, "Agent": "REINFORCE"}, {"env_step": 2119680, "rew": 375.6939, "rew_std": 88.9606, "Agent": "REINFORCE"}, {"env_step": 2150400, "rew": 378.8614, "rew_std": 87.7811, "Agent": "REINFORCE"}, {"env_step": 2181120, "rew": 379.6722, "rew_std": 79.5336, "Agent": "REINFORCE"}, {"env_step": 2211840, "rew": 377.2777, "rew_std": 89.5931, "Agent": "REINFORCE"}, {"env_step": 2242560, "rew": 369.9105, "rew_std": 86.6011, "Agent": "REINFORCE"}, {"env_step": 2273280, "rew": 377.3181, "rew_std": 71.1586, "Agent": "REINFORCE"}, {"env_step": 2304000, "rew": 382.9155, "rew_std": 76.8354, "Agent": "REINFORCE"}, {"env_step": 2334720, "rew": 386.854, "rew_std": 79.9414, "Agent": "REINFORCE"}, {"env_step": 2365440, "rew": 380.7335, "rew_std": 109.3207, "Agent": "REINFORCE"}, {"env_step": 2396160, "rew": 393.7874, "rew_std": 80.6004, "Agent": "REINFORCE"}, {"env_step": 2426880, "rew": 380.0022, "rew_std": 77.799, "Agent": "REINFORCE"}, {"env_step": 2457600, "rew": 377.9924, "rew_std": 79.3554, "Agent": "REINFORCE"}, {"env_step": 2488320, "rew": 381.3284, "rew_std": 90.5689, "Agent": "REINFORCE"}, {"env_step": 2519040, "rew": 373.0119, "rew_std": 75.9316, "Agent": "REINFORCE"}, {"env_step": 2549760, "rew": 364.0844, "rew_std": 46.8014, "Agent": "REINFORCE"}, {"env_step": 2580480, "rew": 368.931, "rew_std": 42.3915, "Agent": "REINFORCE"}, {"env_step": 2611200, "rew": 365.2996, "rew_std": 34.5441, "Agent": "REINFORCE"}, {"env_step": 2641920, "rew": 375.5965, "rew_std": 51.0352, "Agent": "REINFORCE"}, {"env_step": 2672640, "rew": 366.0311, "rew_std": 54.5752, "Agent": "REINFORCE"}, {"env_step": 2703360, "rew": 375.709, "rew_std": 57.4101, "Agent": "REINFORCE"}, {"env_step": 2734080, "rew": 366.1557, "rew_std": 51.1812, "Agent": "REINFORCE"}, {"env_step": 2764800, "rew": 362.7805, "rew_std": 52.9235, "Agent": "REINFORCE"}, {"env_step": 2795520, "rew": 363.9702, "rew_std": 44.3478, "Agent": "REINFORCE"}, {"env_step": 2826240, "rew": 365.2992, "rew_std": 48.0731, "Agent": "REINFORCE"}, {"env_step": 2856960, "rew": 370.9386, "rew_std": 66.4269, "Agent": "REINFORCE"}, {"env_step": 2887680, "rew": 366.0387, "rew_std": 64.0745, "Agent": "REINFORCE"}, {"env_step": 2918400, "rew": 361.6844, "rew_std": 53.0049, "Agent": "REINFORCE"}, {"env_step": 2949120, "rew": 364.995, "rew_std": 55.0417, "Agent": "REINFORCE"}, {"env_step": 2979840, "rew": 367.1026, "rew_std": 46.2725, "Agent": "REINFORCE"}, {"env_step": 3010560, "rew": 368.1378, "rew_std": 50.5468, "Agent": "REINFORCE"}, {"env_step": 3041280, "rew": 368.6792, "rew_std": 59.7536, "Agent": "REINFORCE"}, {"env_step": 3072000, "rew": 355.6231, "rew_std": 41.6614, "Agent": "REINFORCE"}, {"env_step": 3102720, "rew": 355.8605, "rew_std": 44.1664, "Agent": "REINFORCE"}, {"env_step": 3133440, "rew": 355.502, "rew_std": 44.6819, "Agent": "REINFORCE"}, {"env_step": 3164160, "rew": 351.8672, "rew_std": 46.9554, "Agent": "REINFORCE"}, {"env_step": 3194880, "rew": 354.3732, "rew_std": 39.958, "Agent": "REINFORCE"}, {"env_step": 3225600, "rew": 363.8132, "rew_std": 43.6584, "Agent": "REINFORCE"}, {"env_step": 3256320, "rew": 366.9973, "rew_std": 52.4478, "Agent": "REINFORCE"}, {"env_step": 3287040, "rew": 359.498, "rew_std": 37.2594, "Agent": "REINFORCE"}, {"env_step": 3317760, "rew": 357.1697, "rew_std": 45.6055, "Agent": "REINFORCE"}, {"env_step": 3348480, "rew": 356.0397, "rew_std": 33.5679, "Agent": "REINFORCE"}, {"env_step": 3379200, "rew": 361.9803, "rew_std": 43.0103, "Agent": "REINFORCE"}, {"env_step": 3409920, "rew": 371.8563, "rew_std": 68.2704, "Agent": "REINFORCE"}, {"env_step": 3440640, "rew": 371.9357, "rew_std": 57.6749, "Agent": "REINFORCE"}, {"env_step": 3471360, "rew": 371.5511, "rew_std": 57.979, "Agent": "REINFORCE"}, {"env_step": 3502080, "rew": 356.9314, "rew_std": 40.984, "Agent": "REINFORCE"}, {"env_step": 3532800, "rew": 363.4594, "rew_std": 41.5677, "Agent": "REINFORCE"}, {"env_step": 3563520, "rew": 374.1565, "rew_std": 56.8199, "Agent": "REINFORCE"}, {"env_step": 3594240, "rew": 371.328, "rew_std": 48.9273, "Agent": "REINFORCE"}, {"env_step": 3624960, "rew": 368.6637, "rew_std": 52.6637, "Agent": "REINFORCE"}, {"env_step": 3655680, "rew": 362.5879, "rew_std": 35.8333, "Agent": "REINFORCE"}, {"env_step": 3686400, "rew": 360.6588, "rew_std": 40.4034, "Agent": "REINFORCE"}, {"env_step": 3717120, "rew": 358.0624, "rew_std": 27.6744, "Agent": "REINFORCE"}, {"env_step": 3747840, "rew": 362.8848, "rew_std": 31.2075, "Agent": "REINFORCE"}, {"env_step": 3778560, "rew": 369.7099, "rew_std": 49.5931, "Agent": "REINFORCE"}, {"env_step": 3809280, "rew": 362.1919, "rew_std": 26.8217, "Agent": "REINFORCE"}, {"env_step": 3840000, "rew": 365.3113, "rew_std": 40.6711, "Agent": "REINFORCE"}, {"env_step": 3870720, "rew": 370.9763, "rew_std": 51.2981, "Agent": "REINFORCE"}, {"env_step": 3901440, "rew": 394.1719, "rew_std": 79.1728, "Agent": "REINFORCE"}, {"env_step": 3932160, "rew": 409.7305, "rew_std": 88.1129, "Agent": "REINFORCE"}, {"env_step": 3962880, "rew": 405.7927, "rew_std": 94.4966, "Agent": "REINFORCE"}, {"env_step": 3993600, "rew": 405.7145, "rew_std": 98.6886, "Agent": "REINFORCE"}, {"env_step": 4024320, "rew": 400.252, "rew_std": 91.7664, "Agent": "REINFORCE"}, {"env_step": 4055040, "rew": 408.2449, "rew_std": 108.9827, "Agent": "REINFORCE"}, {"env_step": 4085760, "rew": 397.0612, "rew_std": 97.8046, "Agent": "REINFORCE"}, {"env_step": 4116480, "rew": 402.881, "rew_std": 99.515, "Agent": "REINFORCE"}, {"env_step": 4147200, "rew": 415.9558, "rew_std": 104.7377, "Agent": "REINFORCE"}, {"env_step": 4177920, "rew": 391.8573, "rew_std": 94.7846, "Agent": "REINFORCE"}, {"env_step": 4208640, "rew": 388.7508, "rew_std": 89.8436, "Agent": "REINFORCE"}, {"env_step": 4239360, "rew": 392.2457, "rew_std": 103.2659, "Agent": "REINFORCE"}, {"env_step": 4270080, "rew": 386.7837, "rew_std": 90.8458, "Agent": "REINFORCE"}, {"env_step": 4300800, "rew": 387.743, "rew_std": 103.9218, "Agent": "REINFORCE"}, {"env_step": 4331520, "rew": 400.4464, "rew_std": 102.1787, "Agent": "REINFORCE"}, {"env_step": 4362240, "rew": 392.1932, "rew_std": 87.3258, "Agent": "REINFORCE"}, {"env_step": 4392960, "rew": 396.4165, "rew_std": 105.8792, "Agent": "REINFORCE"}, {"env_step": 4423680, "rew": 387.7544, "rew_std": 97.0025, "Agent": "REINFORCE"}, {"env_step": 4454400, "rew": 391.0999, "rew_std": 98.9182, "Agent": "REINFORCE"}, {"env_step": 4485120, "rew": 391.6717, "rew_std": 102.3208, "Agent": "REINFORCE"}, {"env_step": 4515840, "rew": 392.4672, "rew_std": 111.5205, "Agent": "REINFORCE"}, {"env_step": 4546560, "rew": 382.6473, "rew_std": 97.9509, "Agent": "REINFORCE"}, {"env_step": 4577280, "rew": 385.3088, "rew_std": 96.1275, "Agent": "REINFORCE"}, {"env_step": 4608000, "rew": 383.38, "rew_std": 97.7575, "Agent": "REINFORCE"}, {"env_step": 4638720, "rew": 382.9842, "rew_std": 94.2347, "Agent": "REINFORCE"}, {"env_step": 4669440, "rew": 399.8325, "rew_std": 105.9479, "Agent": "REINFORCE"}, {"env_step": 4700160, "rew": 390.9329, "rew_std": 98.4279, "Agent": "REINFORCE"}, {"env_step": 4730880, "rew": 409.3433, "rew_std": 118.3284, "Agent": "REINFORCE"}, {"env_step": 4761600, "rew": 394.2645, "rew_std": 107.2525, "Agent": "REINFORCE"}, {"env_step": 4792320, "rew": 384.1185, "rew_std": 102.1598, "Agent": "REINFORCE"}, {"env_step": 4823040, "rew": 396.9076, "rew_std": 97.2542, "Agent": "REINFORCE"}, {"env_step": 4853760, "rew": 390.4801, "rew_std": 99.5176, "Agent": "REINFORCE"}, {"env_step": 4884480, "rew": 389.4399, "rew_std": 92.1015, "Agent": "REINFORCE"}, {"env_step": 4915200, "rew": 394.0313, "rew_std": 101.7443, "Agent": "REINFORCE"}, {"env_step": 4945920, "rew": 390.3732, "rew_std": 91.6181, "Agent": "REINFORCE"}, {"env_step": 4976640, "rew": 383.9362, "rew_std": 91.6545, "Agent": "REINFORCE"}, {"env_step": 5007360, "rew": 377.8827, "rew_std": 92.611, "Agent": "REINFORCE"}, {"env_step": 5038080, "rew": 381.0909, "rew_std": 95.6623, "Agent": "REINFORCE"}, {"env_step": 5068800, "rew": 386.0676, "rew_std": 85.6938, "Agent": "REINFORCE"}, {"env_step": 5099520, "rew": 374.0582, "rew_std": 62.9661, "Agent": "REINFORCE"}, {"env_step": 5130240, "rew": 382.5204, "rew_std": 67.5511, "Agent": "REINFORCE"}, {"env_step": 5160960, "rew": 385.7647, "rew_std": 81.6735, "Agent": "REINFORCE"}, {"env_step": 5191680, "rew": 385.5587, "rew_std": 79.1534, "Agent": "REINFORCE"}, {"env_step": 5222400, "rew": 375.1411, "rew_std": 89.7856, "Agent": "REINFORCE"}, {"env_step": 5253120, "rew": 382.3445, "rew_std": 96.1506, "Agent": "REINFORCE"}, {"env_step": 5283840, "rew": 392.8232, "rew_std": 121.0017, "Agent": "REINFORCE"}, {"env_step": 5314560, "rew": 381.6725, "rew_std": 95.3916, "Agent": "REINFORCE"}, {"env_step": 5345280, "rew": 390.9818, "rew_std": 104.5308, "Agent": "REINFORCE"}, {"env_step": 5376000, "rew": 384.3439, "rew_std": 106.4543, "Agent": "REINFORCE"}, {"env_step": 5406720, "rew": 378.383, "rew_std": 107.0747, "Agent": "REINFORCE"}, {"env_step": 5437440, "rew": 380.5131, "rew_std": 112.1825, "Agent": "REINFORCE"}, {"env_step": 5468160, "rew": 381.4596, "rew_std": 116.9546, "Agent": "REINFORCE"}, {"env_step": 5498880, "rew": 376.408, "rew_std": 109.4098, "Agent": "REINFORCE"}, {"env_step": 5529600, "rew": 371.7655, "rew_std": 94.5637, "Agent": "REINFORCE"}, {"env_step": 5560320, "rew": 371.3834, "rew_std": 99.8961, "Agent": "REINFORCE"}, {"env_step": 5591040, "rew": 373.4715, "rew_std": 105.4585, "Agent": "REINFORCE"}, {"env_step": 5621760, "rew": 379.2169, "rew_std": 114.6349, "Agent": "REINFORCE"}, {"env_step": 5652480, "rew": 377.9405, "rew_std": 107.3001, "Agent": "REINFORCE"}, {"env_step": 5683200, "rew": 373.9021, "rew_std": 97.6858, "Agent": "REINFORCE"}, {"env_step": 5713920, "rew": 375.604, "rew_std": 105.6384, "Agent": "REINFORCE"}, {"env_step": 5744640, "rew": 379.1034, "rew_std": 103.1234, "Agent": "REINFORCE"}, {"env_step": 5775360, "rew": 381.6423, "rew_std": 100.8259, "Agent": "REINFORCE"}, {"env_step": 5806080, "rew": 378.8504, "rew_std": 104.2578, "Agent": "REINFORCE"}, {"env_step": 5836800, "rew": 382.0778, "rew_std": 110.7984, "Agent": "REINFORCE"}, {"env_step": 5867520, "rew": 381.9858, "rew_std": 107.0871, "Agent": "REINFORCE"}, {"env_step": 5898240, "rew": 380.0554, "rew_std": 109.317, "Agent": "REINFORCE"}, {"env_step": 5928960, "rew": 381.0922, "rew_std": 101.8419, "Agent": "REINFORCE"}, {"env_step": 5959680, "rew": 378.747, "rew_std": 88.6599, "Agent": "REINFORCE"}, {"env_step": 5990400, "rew": 392.1596, "rew_std": 94.625, "Agent": "REINFORCE"}, {"env_step": 6021120, "rew": 388.2924, "rew_std": 106.6063, "Agent": "REINFORCE"}, {"env_step": 6051840, "rew": 391.8991, "rew_std": 105.9083, "Agent": "REINFORCE"}, {"env_step": 6082560, "rew": 395.3478, "rew_std": 106.1762, "Agent": "REINFORCE"}, {"env_step": 6113280, "rew": 402.5229, "rew_std": 118.2928, "Agent": "REINFORCE"}, {"env_step": 6144000, "rew": 402.5777, "rew_std": 123.3479, "Agent": "REINFORCE"}, {"env_step": 6174720, "rew": 409.2375, "rew_std": 123.5333, "Agent": "REINFORCE"}, {"env_step": 6205440, "rew": 401.1756, "rew_std": 105.6258, "Agent": "REINFORCE"}, {"env_step": 6236160, "rew": 393.5145, "rew_std": 103.74, "Agent": "REINFORCE"}, {"env_step": 6266880, "rew": 392.5335, "rew_std": 97.7167, "Agent": "REINFORCE"}, {"env_step": 6297600, "rew": 388.9988, "rew_std": 104.4046, "Agent": "REINFORCE"}, {"env_step": 6328320, "rew": 400.0679, "rew_std": 111.9244, "Agent": "REINFORCE"}, {"env_step": 6359040, "rew": 399.2567, "rew_std": 112.5052, "Agent": "REINFORCE"}, {"env_step": 6389760, "rew": 397.178, "rew_std": 109.4325, "Agent": "REINFORCE"}, {"env_step": 6420480, "rew": 397.1731, "rew_std": 95.5987, "Agent": "REINFORCE"}, {"env_step": 6451200, "rew": 405.8894, "rew_std": 102.3605, "Agent": "REINFORCE"}, {"env_step": 6481920, "rew": 399.0744, "rew_std": 108.8508, "Agent": "REINFORCE"}, {"env_step": 6512640, "rew": 405.8708, "rew_std": 112.578, "Agent": "REINFORCE"}, {"env_step": 6543360, "rew": 398.2285, "rew_std": 94.2502, "Agent": "REINFORCE"}, {"env_step": 6574080, "rew": 404.914, "rew_std": 96.5741, "Agent": "REINFORCE"}, {"env_step": 6604800, "rew": 393.1239, "rew_std": 84.9309, "Agent": "REINFORCE"}, {"env_step": 6635520, "rew": 404.68, "rew_std": 102.7677, "Agent": "REINFORCE"}, {"env_step": 6666240, "rew": 389.5942, "rew_std": 79.4133, "Agent": "REINFORCE"}, {"env_step": 6696960, "rew": 392.5762, "rew_std": 85.0446, "Agent": "REINFORCE"}, {"env_step": 6727680, "rew": 399.4966, "rew_std": 93.1352, "Agent": "REINFORCE"}, {"env_step": 6758400, "rew": 399.9873, "rew_std": 99.9946, "Agent": "REINFORCE"}, {"env_step": 6789120, "rew": 393.2349, "rew_std": 92.2259, "Agent": "REINFORCE"}, {"env_step": 6819840, "rew": 387.7151, "rew_std": 88.0488, "Agent": "REINFORCE"}, {"env_step": 6850560, "rew": 386.5385, "rew_std": 84.4063, "Agent": "REINFORCE"}, {"env_step": 6881280, "rew": 388.8521, "rew_std": 74.5801, "Agent": "REINFORCE"}, {"env_step": 6912000, "rew": 398.1654, "rew_std": 90.6041, "Agent": "REINFORCE"}, {"env_step": 6942720, "rew": 384.939, "rew_std": 71.2728, "Agent": "REINFORCE"}, {"env_step": 6973440, "rew": 390.8309, "rew_std": 85.7066, "Agent": "REINFORCE"}, {"env_step": 7004160, "rew": 399.2637, "rew_std": 97.6472, "Agent": "REINFORCE"}, {"env_step": 7034880, "rew": 389.195, "rew_std": 81.2926, "Agent": "REINFORCE"}, {"env_step": 7065600, "rew": 398.1003, "rew_std": 93.8406, "Agent": "REINFORCE"}, {"env_step": 7096320, "rew": 396.5792, "rew_std": 99.5352, "Agent": "REINFORCE"}, {"env_step": 7127040, "rew": 388.7145, "rew_std": 90.0708, "Agent": "REINFORCE"}, {"env_step": 7157760, "rew": 389.8861, "rew_std": 85.8735, "Agent": "REINFORCE"}, {"env_step": 7188480, "rew": 390.0967, "rew_std": 82.3581, "Agent": "REINFORCE"}, {"env_step": 7219200, "rew": 379.0566, "rew_std": 64.0611, "Agent": "REINFORCE"}, {"env_step": 7249920, "rew": 385.6624, "rew_std": 81.0679, "Agent": "REINFORCE"}, {"env_step": 7280640, "rew": 386.8548, "rew_std": 75.6701, "Agent": "REINFORCE"}, {"env_step": 7311360, "rew": 379.9336, "rew_std": 77.4284, "Agent": "REINFORCE"}, {"env_step": 7342080, "rew": 379.5128, "rew_std": 75.1445, "Agent": "REINFORCE"}, {"env_step": 7372800, "rew": 374.7474, "rew_std": 58.1994, "Agent": "REINFORCE"}, {"env_step": 7403520, "rew": 385.7565, "rew_std": 75.5155, "Agent": "REINFORCE"}, {"env_step": 7434240, "rew": 379.3197, "rew_std": 67.6012, "Agent": "REINFORCE"}, {"env_step": 7464960, "rew": 375.6075, "rew_std": 62.7015, "Agent": "REINFORCE"}, {"env_step": 7495680, "rew": 374.1764, "rew_std": 54.2789, "Agent": "REINFORCE"}, {"env_step": 7526400, "rew": 392.6875, "rew_std": 68.5086, "Agent": "REINFORCE"}, {"env_step": 7557120, "rew": 382.8261, "rew_std": 63.2083, "Agent": "REINFORCE"}, {"env_step": 7587840, "rew": 385.8388, "rew_std": 72.5709, "Agent": "REINFORCE"}, {"env_step": 7618560, "rew": 392.9784, "rew_std": 84.3356, "Agent": "REINFORCE"}, {"env_step": 7649280, "rew": 390.1091, "rew_std": 83.1857, "Agent": "REINFORCE"}, {"env_step": 7680000, "rew": 386.6012, "rew_std": 80.7424, "Agent": "REINFORCE"}, {"env_step": 7710720, "rew": 388.8892, "rew_std": 63.1784, "Agent": "REINFORCE"}, {"env_step": 7741440, "rew": 382.1974, "rew_std": 68.2579, "Agent": "REINFORCE"}, {"env_step": 7772160, "rew": 379.7919, "rew_std": 59.8642, "Agent": "REINFORCE"}, {"env_step": 7802880, "rew": 387.753, "rew_std": 63.2395, "Agent": "REINFORCE"}, {"env_step": 7833600, "rew": 387.4659, "rew_std": 64.6213, "Agent": "REINFORCE"}, {"env_step": 7864320, "rew": 379.2728, "rew_std": 85.6654, "Agent": "REINFORCE"}, {"env_step": 7895040, "rew": 387.6978, "rew_std": 71.3842, "Agent": "REINFORCE"}, {"env_step": 7925760, "rew": 387.5727, "rew_std": 75.2939, "Agent": "REINFORCE"}, {"env_step": 7956480, "rew": 388.9357, "rew_std": 72.2209, "Agent": "REINFORCE"}, {"env_step": 7987200, "rew": 387.5855, "rew_std": 73.3018, "Agent": "REINFORCE"}, {"env_step": 8017920, "rew": 383.1652, "rew_std": 72.855, "Agent": "REINFORCE"}, {"env_step": 8048640, "rew": 396.8474, "rew_std": 84.074, "Agent": "REINFORCE"}, {"env_step": 8079360, "rew": 383.3162, "rew_std": 73.0269, "Agent": "REINFORCE"}, {"env_step": 8110080, "rew": 387.3427, "rew_std": 74.5757, "Agent": "REINFORCE"}, {"env_step": 8140800, "rew": 391.8142, "rew_std": 60.3373, "Agent": "REINFORCE"}, {"env_step": 8171520, "rew": 384.0898, "rew_std": 65.3742, "Agent": "REINFORCE"}, {"env_step": 8202240, "rew": 388.1516, "rew_std": 71.4981, "Agent": "REINFORCE"}, {"env_step": 8232960, "rew": 386.9834, "rew_std": 66.7738, "Agent": "REINFORCE"}, {"env_step": 8263680, "rew": 393.1251, "rew_std": 89.5124, "Agent": "REINFORCE"}, {"env_step": 8294400, "rew": 390.7711, "rew_std": 69.4544, "Agent": "REINFORCE"}, {"env_step": 8325120, "rew": 390.4361, "rew_std": 74.7419, "Agent": "REINFORCE"}, {"env_step": 8355840, "rew": 384.8838, "rew_std": 74.9661, "Agent": "REINFORCE"}, {"env_step": 8386560, "rew": 387.1014, "rew_std": 73.4848, "Agent": "REINFORCE"}, {"env_step": 8417280, "rew": 385.2746, "rew_std": 65.9136, "Agent": "REINFORCE"}, {"env_step": 8448000, "rew": 389.9486, "rew_std": 82.8964, "Agent": "REINFORCE"}, {"env_step": 8478720, "rew": 381.0628, "rew_std": 74.0303, "Agent": "REINFORCE"}, {"env_step": 8509440, "rew": 396.4438, "rew_std": 81.3918, "Agent": "REINFORCE"}, {"env_step": 8540160, "rew": 387.083, "rew_std": 70.84, "Agent": "REINFORCE"}, {"env_step": 8570880, "rew": 384.714, "rew_std": 71.8535, "Agent": "REINFORCE"}, {"env_step": 8601600, "rew": 387.3163, "rew_std": 73.4316, "Agent": "REINFORCE"}, {"env_step": 8632320, "rew": 376.7846, "rew_std": 66.9371, "Agent": "REINFORCE"}, {"env_step": 8663040, "rew": 389.0855, "rew_std": 66.3918, "Agent": "REINFORCE"}, {"env_step": 8693760, "rew": 386.6858, "rew_std": 74.6856, "Agent": "REINFORCE"}, {"env_step": 8724480, "rew": 380.3388, "rew_std": 62.0064, "Agent": "REINFORCE"}, {"env_step": 8755200, "rew": 391.8039, "rew_std": 70.9114, "Agent": "REINFORCE"}, {"env_step": 8785920, "rew": 385.2417, "rew_std": 81.3082, "Agent": "REINFORCE"}, {"env_step": 8816640, "rew": 382.185, "rew_std": 67.3521, "Agent": "REINFORCE"}, {"env_step": 8847360, "rew": 390.1358, "rew_std": 68.1274, "Agent": "REINFORCE"}, {"env_step": 8878080, "rew": 392.2323, "rew_std": 79.9112, "Agent": "REINFORCE"}, {"env_step": 8908800, "rew": 385.8885, "rew_std": 59.5361, "Agent": "REINFORCE"}, {"env_step": 8939520, "rew": 390.0189, "rew_std": 75.2109, "Agent": "REINFORCE"}, {"env_step": 8970240, "rew": 389.2995, "rew_std": 78.8738, "Agent": "REINFORCE"}, {"env_step": 9000960, "rew": 389.9112, "rew_std": 77.0676, "Agent": "REINFORCE"}, {"env_step": 9031680, "rew": 390.7582, "rew_std": 61.6335, "Agent": "REINFORCE"}, {"env_step": 9062400, "rew": 386.58, "rew_std": 67.7454, "Agent": "REINFORCE"}, {"env_step": 9093120, "rew": 393.7479, "rew_std": 84.891, "Agent": "REINFORCE"}, {"env_step": 9123840, "rew": 387.8405, "rew_std": 79.688, "Agent": "REINFORCE"}, {"env_step": 9154560, "rew": 389.0189, "rew_std": 72.7803, "Agent": "REINFORCE"}, {"env_step": 9185280, "rew": 393.9931, "rew_std": 89.404, "Agent": "REINFORCE"}, {"env_step": 9216000, "rew": 391.9484, "rew_std": 81.6346, "Agent": "REINFORCE"}, {"env_step": 9246720, "rew": 391.9803, "rew_std": 83.1838, "Agent": "REINFORCE"}, {"env_step": 9277440, "rew": 393.2402, "rew_std": 80.7977, "Agent": "REINFORCE"}, {"env_step": 9308160, "rew": 388.6211, "rew_std": 65.8222, "Agent": "REINFORCE"}, {"env_step": 9338880, "rew": 386.8091, "rew_std": 65.9946, "Agent": "REINFORCE"}, {"env_step": 9369600, "rew": 393.0005, "rew_std": 82.596, "Agent": "REINFORCE"}, {"env_step": 9400320, "rew": 398.6098, "rew_std": 77.0395, "Agent": "REINFORCE"}, {"env_step": 9431040, "rew": 387.6794, "rew_std": 68.2785, "Agent": "REINFORCE"}, {"env_step": 9461760, "rew": 396.2232, "rew_std": 84.8934, "Agent": "REINFORCE"}, {"env_step": 9492480, "rew": 393.7139, "rew_std": 72.3839, "Agent": "REINFORCE"}, {"env_step": 9523200, "rew": 386.8038, "rew_std": 78.5513, "Agent": "REINFORCE"}, {"env_step": 9553920, "rew": 398.665, "rew_std": 83.0765, "Agent": "REINFORCE"}, {"env_step": 9584640, "rew": 394.106, "rew_std": 81.1441, "Agent": "REINFORCE"}, {"env_step": 9615360, "rew": 394.6446, "rew_std": 80.3958, "Agent": "REINFORCE"}, {"env_step": 9646080, "rew": 394.4672, "rew_std": 92.5547, "Agent": "REINFORCE"}, {"env_step": 9676800, "rew": 394.4883, "rew_std": 85.8432, "Agent": "REINFORCE"}, {"env_step": 9707520, "rew": 383.2617, "rew_std": 62.3773, "Agent": "REINFORCE"}, {"env_step": 9738240, "rew": 393.0799, "rew_std": 88.1284, "Agent": "REINFORCE"}, {"env_step": 9768960, "rew": 393.9025, "rew_std": 91.0352, "Agent": "REINFORCE"}, {"env_step": 9799680, "rew": 402.741, "rew_std": 85.7434, "Agent": "REINFORCE"}, {"env_step": 9830400, "rew": 390.5062, "rew_std": 64.6333, "Agent": "REINFORCE"}, {"env_step": 9861120, "rew": 395.8816, "rew_std": 81.3256, "Agent": "REINFORCE"}, {"env_step": 9891840, "rew": 389.786, "rew_std": 75.6932, "Agent": "REINFORCE"}, {"env_step": 9922560, "rew": 392.9576, "rew_std": 76.6835, "Agent": "REINFORCE"}, {"env_step": 9953280, "rew": 386.0522, "rew_std": 74.3712, "Agent": "REINFORCE"}, {"env_step": 9984000, "rew": 387.3965, "rew_std": 70.1528, "Agent": "REINFORCE"}, {"env_step": 10014720, "rew": 395.1271, "rew_std": 97.8375, "Agent": "REINFORCE"}, {"env_step": 10045440, "rew": 395.2879, "rew_std": 74.4887, "Agent": "REINFORCE"}, {"env_step": 10076160, "rew": 395.3669, "rew_std": 83.4544, "Agent": "REINFORCE"}, {"env_step": 10106880, "rew": 378.1233, "rew_std": 66.645, "Agent": "REINFORCE"}, {"env_step": 10137600, "rew": 390.0436, "rew_std": 78.3602, "Agent": "REINFORCE"}, {"env_step": 10168320, "rew": 390.2037, "rew_std": 83.6567, "Agent": "REINFORCE"}, {"env_step": 10199040, "rew": 384.08, "rew_std": 66.8607, "Agent": "REINFORCE"}, {"env_step": 10229760, "rew": 391.8881, "rew_std": 86.4776, "Agent": "REINFORCE"}, {"env_step": 0, "rew": 17.3274, "rew_std": 3.1882, "Agent": "A2C"}, {"env_step": 30000, "rew": 462.4966, "rew_std": 346.8055, "Agent": "A2C"}, {"env_step": 60000, "rew": 622.0757, "rew_std": 308.1571, "Agent": "A2C"}, {"env_step": 90000, "rew": 682.6425, "rew_std": 293.7176, "Agent": "A2C"}, {"env_step": 120000, "rew": 626.6831, "rew_std": 267.0181, "Agent": "A2C"}, {"env_step": 150000, "rew": 588.8864, "rew_std": 272.5663, "Agent": "A2C"}, {"env_step": 180000, "rew": 570.3548, "rew_std": 273.4997, "Agent": "A2C"}, {"env_step": 210000, "rew": 583.2997, "rew_std": 269.3662, "Agent": "A2C"}, {"env_step": 240000, "rew": 568.1374, "rew_std": 285.7505, "Agent": "A2C"}, {"env_step": 270000, "rew": 600.4265, "rew_std": 283.2298, "Agent": "A2C"}, {"env_step": 300000, "rew": 581.2122, "rew_std": 281.8224, "Agent": "A2C"}, {"env_step": 330000, "rew": 582.0573, "rew_std": 275.9559, "Agent": "A2C"}, {"env_step": 360000, "rew": 639.7437, "rew_std": 248.9154, "Agent": "A2C"}, {"env_step": 390000, "rew": 748.2057, "rew_std": 324.7842, "Agent": "A2C"}, {"env_step": 420000, "rew": 826.1403, "rew_std": 476.4329, "Agent": "A2C"}, {"env_step": 450000, "rew": 772.2532, "rew_std": 355.8575, "Agent": "A2C"}, {"env_step": 480000, "rew": 729.2479, "rew_std": 296.4466, "Agent": "A2C"}, {"env_step": 510000, "rew": 819.3713, "rew_std": 456.3531, "Agent": "A2C"}, {"env_step": 540000, "rew": 709.3754, "rew_std": 292.8698, "Agent": "A2C"}, {"env_step": 570000, "rew": 732.1387, "rew_std": 328.7899, "Agent": "A2C"}, {"env_step": 600000, "rew": 990.2754, "rew_std": 488.759, "Agent": "A2C"}, {"env_step": 630000, "rew": 982.0049, "rew_std": 340.0413, "Agent": "A2C"}, {"env_step": 660000, "rew": 940.6046, "rew_std": 400.8275, "Agent": "A2C"}, {"env_step": 690000, "rew": 951.6317, "rew_std": 224.9542, "Agent": "A2C"}, {"env_step": 720000, "rew": 1166.5681, "rew_std": 264.526, "Agent": "A2C"}, {"env_step": 750000, "rew": 1253.1615, "rew_std": 458.0398, "Agent": "A2C"}, {"env_step": 780000, "rew": 1106.3958, "rew_std": 266.184, "Agent": "A2C"}, {"env_step": 810000, "rew": 1037.3397, "rew_std": 192.7324, "Agent": "A2C"}, {"env_step": 840000, "rew": 1194.818, "rew_std": 557.2156, "Agent": "A2C"}, {"env_step": 870000, "rew": 1174.3561, "rew_std": 222.2454, "Agent": "A2C"}, {"env_step": 900000, "rew": 1212.1007, "rew_std": 340.1215, "Agent": "A2C"}, {"env_step": 930000, "rew": 1132.2114, "rew_std": 514.8548, "Agent": "A2C"}, {"env_step": 960000, "rew": 1201.6032, "rew_std": 580.1104, "Agent": "A2C"}, {"env_step": 990000, "rew": 1223.54, "rew_std": 339.8763, "Agent": "A2C"}, {"env_step": 1020000, "rew": 954.8797, "rew_std": 314.4283, "Agent": "A2C"}, {"env_step": 1050000, "rew": 1160.6148, "rew_std": 429.8111, "Agent": "A2C"}, {"env_step": 1080000, "rew": 1034.2824, "rew_std": 408.4905, "Agent": "A2C"}, {"env_step": 1110000, "rew": 990.1428, "rew_std": 453.6877, "Agent": "A2C"}, {"env_step": 1140000, "rew": 1146.8304, "rew_std": 373.1246, "Agent": "A2C"}, {"env_step": 1170000, "rew": 1197.3747, "rew_std": 623.5087, "Agent": "A2C"}, {"env_step": 1200000, "rew": 1165.3104, "rew_std": 555.5445, "Agent": "A2C"}, {"env_step": 1230000, "rew": 1368.01, "rew_std": 586.2133, "Agent": "A2C"}, {"env_step": 1260000, "rew": 1345.0412, "rew_std": 559.5705, "Agent": "A2C"}, {"env_step": 1290000, "rew": 1175.3513, "rew_std": 607.5811, "Agent": "A2C"}, {"env_step": 1320000, "rew": 1094.7432, "rew_std": 688.7237, "Agent": "A2C"}, {"env_step": 1350000, "rew": 1296.5528, "rew_std": 737.2834, "Agent": "A2C"}, {"env_step": 1380000, "rew": 1088.8646, "rew_std": 336.7799, "Agent": "A2C"}, {"env_step": 1410000, "rew": 1354.1653, "rew_std": 371.5432, "Agent": "A2C"}, {"env_step": 1440000, "rew": 1275.0429, "rew_std": 374.1279, "Agent": "A2C"}, {"env_step": 1470000, "rew": 1553.9737, "rew_std": 726.4136, "Agent": "A2C"}, {"env_step": 1500000, "rew": 1348.2717, "rew_std": 537.4082, "Agent": "A2C"}, {"env_step": 1530000, "rew": 1196.8302, "rew_std": 579.4114, "Agent": "A2C"}, {"env_step": 1560000, "rew": 1301.034, "rew_std": 435.1382, "Agent": "A2C"}, {"env_step": 1590000, "rew": 1381.2474, "rew_std": 586.198, "Agent": "A2C"}, {"env_step": 1620000, "rew": 1210.7781, "rew_std": 452.1103, "Agent": "A2C"}, {"env_step": 1650000, "rew": 1159.8688, "rew_std": 308.3852, "Agent": "A2C"}, {"env_step": 1680000, "rew": 1233.043, "rew_std": 448.2399, "Agent": "A2C"}, {"env_step": 1710000, "rew": 1413.6957, "rew_std": 452.6843, "Agent": "A2C"}, {"env_step": 1740000, "rew": 1427.6175, "rew_std": 536.558, "Agent": "A2C"}, {"env_step": 1770000, "rew": 1435.2136, "rew_std": 702.4925, "Agent": "A2C"}, {"env_step": 1800000, "rew": 1517.1957, "rew_std": 570.5674, "Agent": "A2C"}, {"env_step": 1830000, "rew": 1091.752, "rew_std": 361.7346, "Agent": "A2C"}, {"env_step": 1860000, "rew": 1199.0044, "rew_std": 400.4895, "Agent": "A2C"}, {"env_step": 1890000, "rew": 1292.4859, "rew_std": 560.0479, "Agent": "A2C"}, {"env_step": 1920000, "rew": 1419.6097, "rew_std": 639.8421, "Agent": "A2C"}, {"env_step": 1950000, "rew": 1415.1152, "rew_std": 413.4579, "Agent": "A2C"}, {"env_step": 1980000, "rew": 1176.76, "rew_std": 481.8063, "Agent": "A2C"}, {"env_step": 2010000, "rew": 1294.9907, "rew_std": 429.3094, "Agent": "A2C"}, {"env_step": 2040000, "rew": 1456.2192, "rew_std": 462.3628, "Agent": "A2C"}, {"env_step": 2070000, "rew": 1608.6202, "rew_std": 529.515, "Agent": "A2C"}, {"env_step": 2100000, "rew": 1526.9954, "rew_std": 652.4756, "Agent": "A2C"}, {"env_step": 2130000, "rew": 1505.7459, "rew_std": 656.8452, "Agent": "A2C"}, {"env_step": 2160000, "rew": 1504.5428, "rew_std": 594.6738, "Agent": "A2C"}, {"env_step": 2190000, "rew": 1457.4778, "rew_std": 556.3529, "Agent": "A2C"}, {"env_step": 2220000, "rew": 1376.9743, "rew_std": 396.4713, "Agent": "A2C"}, {"env_step": 2250000, "rew": 1314.4171, "rew_std": 455.926, "Agent": "A2C"}, {"env_step": 2280000, "rew": 1181.3131, "rew_std": 421.3108, "Agent": "A2C"}, {"env_step": 2310000, "rew": 1327.9512, "rew_std": 548.2671, "Agent": "A2C"}, {"env_step": 2340000, "rew": 1214.5111, "rew_std": 312.23, "Agent": "A2C"}, {"env_step": 2370000, "rew": 1340.6044, "rew_std": 368.7874, "Agent": "A2C"}, {"env_step": 2400000, "rew": 1381.3645, "rew_std": 377.5288, "Agent": "A2C"}, {"env_step": 2430000, "rew": 1409.8832, "rew_std": 362.9671, "Agent": "A2C"}, {"env_step": 2460000, "rew": 1313.9969, "rew_std": 244.2689, "Agent": "A2C"}, {"env_step": 2490000, "rew": 1276.3597, "rew_std": 417.0242, "Agent": "A2C"}, {"env_step": 2520000, "rew": 1349.8452, "rew_std": 397.339, "Agent": "A2C"}, {"env_step": 2550000, "rew": 1257.4598, "rew_std": 450.0145, "Agent": "A2C"}, {"env_step": 2580000, "rew": 1514.8774, "rew_std": 458.0025, "Agent": "A2C"}, {"env_step": 2610000, "rew": 1338.5812, "rew_std": 343.2712, "Agent": "A2C"}, {"env_step": 2640000, "rew": 1441.4321, "rew_std": 484.9153, "Agent": "A2C"}, {"env_step": 2670000, "rew": 1430.9547, "rew_std": 335.1838, "Agent": "A2C"}, {"env_step": 2700000, "rew": 1270.3224, "rew_std": 239.8125, "Agent": "A2C"}, {"env_step": 2730000, "rew": 1402.5757, "rew_std": 358.1328, "Agent": "A2C"}, {"env_step": 2760000, "rew": 1359.1978, "rew_std": 284.1278, "Agent": "A2C"}, {"env_step": 2790000, "rew": 1311.6115, "rew_std": 235.8307, "Agent": "A2C"}, {"env_step": 2820000, "rew": 1402.1925, "rew_std": 478.4434, "Agent": "A2C"}, {"env_step": 2850000, "rew": 1220.61, "rew_std": 366.7868, "Agent": "A2C"}, {"env_step": 2880000, "rew": 1300.8353, "rew_std": 249.6865, "Agent": "A2C"}, {"env_step": 2910000, "rew": 1404.9237, "rew_std": 394.3524, "Agent": "A2C"}, {"env_step": 2940000, "rew": 1429.0835, "rew_std": 425.5458, "Agent": "A2C"}, {"env_step": 2970000, "rew": 1367.0696, "rew_std": 392.2221, "Agent": "A2C"}, {"env_step": 3000000, "rew": 1291.0612, "rew_std": 336.8317, "Agent": "A2C"}] \ No newline at end of file diff --git a/examples/mujoco/benchmark/Humanoid-v3/result.json b/examples/mujoco/benchmark/Humanoid-v3/result.json new file mode 100644 index 0000000..9bd5d28 --- /dev/null +++ b/examples/mujoco/benchmark/Humanoid-v3/result.json @@ -0,0 +1 @@ +[{"env_step": 0, "rew": 177.3211, "rew_std": 77.5947, "Agent": "SAC"}, {"env_step": 5000, "rew": 283.2922, "rew_std": 15.6399, "Agent": "SAC"}, {"env_step": 10000, "rew": 299.9204, "rew_std": 31.8772, "Agent": "SAC"}, {"env_step": 15000, "rew": 435.967, "rew_std": 59.9478, "Agent": "SAC"}, {"env_step": 20000, "rew": 449.6315, "rew_std": 50.5159, "Agent": "SAC"}, {"env_step": 25000, "rew": 462.7225, "rew_std": 70.3407, "Agent": "SAC"}, {"env_step": 30000, "rew": 527.9205, "rew_std": 62.4189, "Agent": "SAC"}, {"env_step": 35000, "rew": 462.1104, "rew_std": 103.0253, "Agent": "SAC"}, {"env_step": 40000, "rew": 502.3742, "rew_std": 89.7226, "Agent": "SAC"}, {"env_step": 45000, "rew": 508.4995, "rew_std": 71.7847, "Agent": "SAC"}, {"env_step": 50000, "rew": 573.9444, "rew_std": 99.0155, "Agent": "SAC"}, {"env_step": 55000, "rew": 550.6721, "rew_std": 65.356, "Agent": "SAC"}, {"env_step": 60000, "rew": 587.3824, "rew_std": 68.5067, "Agent": "SAC"}, {"env_step": 65000, "rew": 601.1097, "rew_std": 87.2493, "Agent": "SAC"}, {"env_step": 70000, "rew": 600.142, "rew_std": 60.1349, "Agent": "SAC"}, {"env_step": 75000, "rew": 594.4696, "rew_std": 61.1347, "Agent": "SAC"}, {"env_step": 80000, "rew": 590.6473, "rew_std": 71.1667, "Agent": "SAC"}, {"env_step": 85000, "rew": 594.3895, "rew_std": 62.5243, "Agent": "SAC"}, {"env_step": 90000, "rew": 618.5582, "rew_std": 86.4116, "Agent": "SAC"}, {"env_step": 95000, "rew": 644.4827, "rew_std": 102.1717, "Agent": "SAC"}, {"env_step": 100000, "rew": 629.4648, "rew_std": 106.451, "Agent": "SAC"}, {"env_step": 105000, "rew": 633.1271, "rew_std": 68.8717, "Agent": "SAC"}, {"env_step": 110000, "rew": 631.02, "rew_std": 71.8617, "Agent": "SAC"}, {"env_step": 115000, "rew": 584.2877, "rew_std": 59.1475, "Agent": "SAC"}, {"env_step": 120000, "rew": 597.579, "rew_std": 52.4268, "Agent": "SAC"}, {"env_step": 125000, "rew": 622.5235, "rew_std": 61.8045, "Agent": "SAC"}, {"env_step": 130000, "rew": 649.8092, "rew_std": 110.1307, "Agent": "SAC"}, {"env_step": 135000, "rew": 613.3262, "rew_std": 75.1675, "Agent": "SAC"}, {"env_step": 140000, "rew": 645.5737, "rew_std": 97.3171, "Agent": "SAC"}, {"env_step": 145000, "rew": 586.2649, "rew_std": 59.617, "Agent": "SAC"}, {"env_step": 150000, "rew": 625.0649, "rew_std": 79.55, "Agent": "SAC"}, {"env_step": 155000, "rew": 637.2469, "rew_std": 83.9602, "Agent": "SAC"}, {"env_step": 160000, "rew": 620.3274, "rew_std": 89.7393, "Agent": "SAC"}, {"env_step": 165000, "rew": 619.5069, "rew_std": 67.1062, "Agent": "SAC"}, {"env_step": 170000, "rew": 630.8656, "rew_std": 68.1679, "Agent": "SAC"}, {"env_step": 175000, "rew": 682.0407, "rew_std": 99.4304, "Agent": "SAC"}, {"env_step": 180000, "rew": 651.833, "rew_std": 60.173, "Agent": "SAC"}, {"env_step": 185000, "rew": 658.08, "rew_std": 53.9764, "Agent": "SAC"}, {"env_step": 190000, "rew": 680.1662, "rew_std": 106.7121, "Agent": "SAC"}, {"env_step": 195000, "rew": 673.2192, "rew_std": 86.1533, "Agent": "SAC"}, {"env_step": 200000, "rew": 767.8674, "rew_std": 306.8412, "Agent": "SAC"}, {"env_step": 205000, "rew": 720.5668, "rew_std": 146.6656, "Agent": "SAC"}, {"env_step": 210000, "rew": 747.1972, "rew_std": 239.9881, "Agent": "SAC"}, {"env_step": 215000, "rew": 790.9514, "rew_std": 322.2662, "Agent": "SAC"}, {"env_step": 220000, "rew": 817.9376, "rew_std": 308.4281, "Agent": "SAC"}, {"env_step": 225000, "rew": 788.8247, "rew_std": 334.612, "Agent": "SAC"}, {"env_step": 230000, "rew": 882.7069, "rew_std": 506.6814, "Agent": "SAC"}, {"env_step": 235000, "rew": 839.2083, "rew_std": 410.4154, "Agent": "SAC"}, {"env_step": 240000, "rew": 835.0124, "rew_std": 357.7152, "Agent": "SAC"}, {"env_step": 245000, "rew": 1098.2356, "rew_std": 823.3634, "Agent": "SAC"}, {"env_step": 250000, "rew": 718.2728, "rew_std": 111.0831, "Agent": "SAC"}, {"env_step": 255000, "rew": 827.5897, "rew_std": 350.9259, "Agent": "SAC"}, {"env_step": 260000, "rew": 863.331, "rew_std": 285.7051, "Agent": "SAC"}, {"env_step": 265000, "rew": 961.4804, "rew_std": 394.793, "Agent": "SAC"}, {"env_step": 270000, "rew": 974.2971, "rew_std": 384.0907, "Agent": "SAC"}, {"env_step": 275000, "rew": 1315.7752, "rew_std": 1288.8193, "Agent": "SAC"}, {"env_step": 280000, "rew": 1036.5671, "rew_std": 413.9318, "Agent": "SAC"}, {"env_step": 285000, "rew": 928.6535, "rew_std": 325.435, "Agent": "SAC"}, {"env_step": 290000, "rew": 1266.3388, "rew_std": 1018.1861, "Agent": "SAC"}, {"env_step": 295000, "rew": 1888.0376, "rew_std": 1495.7165, "Agent": "SAC"}, {"env_step": 300000, "rew": 2044.8745, "rew_std": 1310.538, "Agent": "SAC"}, {"env_step": 305000, "rew": 1470.7834, "rew_std": 803.1499, "Agent": "SAC"}, {"env_step": 310000, "rew": 1434.7183, "rew_std": 748.1661, "Agent": "SAC"}, {"env_step": 315000, "rew": 1784.8552, "rew_std": 942.3052, "Agent": "SAC"}, {"env_step": 320000, "rew": 2115.5942, "rew_std": 1219.7121, "Agent": "SAC"}, {"env_step": 325000, "rew": 2098.7451, "rew_std": 1303.318, "Agent": "SAC"}, {"env_step": 330000, "rew": 1410.295, "rew_std": 761.9196, "Agent": "SAC"}, {"env_step": 335000, "rew": 1552.3471, "rew_std": 741.9198, "Agent": "SAC"}, {"env_step": 340000, "rew": 2020.108, "rew_std": 1466.2618, "Agent": "SAC"}, {"env_step": 345000, "rew": 1933.2728, "rew_std": 1159.0629, "Agent": "SAC"}, {"env_step": 350000, "rew": 2628.7817, "rew_std": 1552.1157, "Agent": "SAC"}, {"env_step": 355000, "rew": 2216.6806, "rew_std": 1650.5325, "Agent": "SAC"}, {"env_step": 360000, "rew": 2484.4005, "rew_std": 1243.7294, "Agent": "SAC"}, {"env_step": 365000, "rew": 2407.9889, "rew_std": 1347.9589, "Agent": "SAC"}, {"env_step": 370000, "rew": 2311.8579, "rew_std": 1337.9997, "Agent": "SAC"}, {"env_step": 375000, "rew": 2689.6973, "rew_std": 1398.341, "Agent": "SAC"}, {"env_step": 380000, "rew": 2529.8016, "rew_std": 1376.7045, "Agent": "SAC"}, {"env_step": 385000, "rew": 2972.2652, "rew_std": 1538.8946, "Agent": "SAC"}, {"env_step": 390000, "rew": 2250.7697, "rew_std": 1463.934, "Agent": "SAC"}, {"env_step": 395000, "rew": 3235.0548, "rew_std": 1710.5823, "Agent": "SAC"}, {"env_step": 400000, "rew": 2194.9027, "rew_std": 1050.3855, "Agent": "SAC"}, {"env_step": 405000, "rew": 2287.7077, "rew_std": 869.8738, "Agent": "SAC"}, {"env_step": 410000, "rew": 2320.09, "rew_std": 1832.5621, "Agent": "SAC"}, {"env_step": 415000, "rew": 3345.4752, "rew_std": 1581.5452, "Agent": "SAC"}, {"env_step": 420000, "rew": 2544.4616, "rew_std": 1693.8445, "Agent": "SAC"}, {"env_step": 425000, "rew": 2865.4076, "rew_std": 1797.1184, "Agent": "SAC"}, {"env_step": 430000, "rew": 3017.8232, "rew_std": 1433.5879, "Agent": "SAC"}, {"env_step": 435000, "rew": 2893.0204, "rew_std": 1024.7995, "Agent": "SAC"}, {"env_step": 440000, "rew": 3590.2399, "rew_std": 1626.9296, "Agent": "SAC"}, {"env_step": 445000, "rew": 3408.5708, "rew_std": 1331.236, "Agent": "SAC"}, {"env_step": 450000, "rew": 3857.306, "rew_std": 1314.3741, "Agent": "SAC"}, {"env_step": 455000, "rew": 3739.2678, "rew_std": 1672.9652, "Agent": "SAC"}, {"env_step": 460000, "rew": 2776.1559, "rew_std": 1809.8104, "Agent": "SAC"}, {"env_step": 465000, "rew": 3620.809, "rew_std": 1596.7628, "Agent": "SAC"}, {"env_step": 470000, "rew": 3130.615, "rew_std": 1227.9251, "Agent": "SAC"}, {"env_step": 475000, "rew": 3261.6376, "rew_std": 1696.0082, "Agent": "SAC"}, {"env_step": 480000, "rew": 3720.7236, "rew_std": 1627.3018, "Agent": "SAC"}, {"env_step": 485000, "rew": 5117.4884, "rew_std": 345.2388, "Agent": "SAC"}, {"env_step": 490000, "rew": 4224.7134, "rew_std": 1283.841, "Agent": "SAC"}, {"env_step": 495000, "rew": 3923.3252, "rew_std": 1043.406, "Agent": "SAC"}, {"env_step": 500000, "rew": 3513.0907, "rew_std": 1830.6286, "Agent": "SAC"}, {"env_step": 505000, "rew": 4370.6183, "rew_std": 838.9721, "Agent": "SAC"}, {"env_step": 510000, "rew": 3937.4691, "rew_std": 1341.3907, "Agent": "SAC"}, {"env_step": 515000, "rew": 4388.9263, "rew_std": 1050.192, "Agent": "SAC"}, {"env_step": 520000, "rew": 4552.7993, "rew_std": 1075.7803, "Agent": "SAC"}, {"env_step": 525000, "rew": 3348.2774, "rew_std": 1406.7555, "Agent": "SAC"}, {"env_step": 530000, "rew": 3760.4822, "rew_std": 1752.8619, "Agent": "SAC"}, {"env_step": 535000, "rew": 3860.4773, "rew_std": 1670.5976, "Agent": "SAC"}, {"env_step": 540000, "rew": 4280.2371, "rew_std": 1125.5536, "Agent": "SAC"}, {"env_step": 545000, "rew": 4687.5066, "rew_std": 800.7659, "Agent": "SAC"}, {"env_step": 550000, "rew": 4513.5009, "rew_std": 960.5825, "Agent": "SAC"}, {"env_step": 555000, "rew": 4916.996, "rew_std": 1105.08, "Agent": "SAC"}, {"env_step": 560000, "rew": 3862.9368, "rew_std": 1644.8431, "Agent": "SAC"}, {"env_step": 565000, "rew": 4770.655, "rew_std": 881.5879, "Agent": "SAC"}, {"env_step": 570000, "rew": 4187.2924, "rew_std": 1616.1352, "Agent": "SAC"}, {"env_step": 575000, "rew": 4504.8732, "rew_std": 1389.8827, "Agent": "SAC"}, {"env_step": 580000, "rew": 4440.6944, "rew_std": 1185.3543, "Agent": "SAC"}, {"env_step": 585000, "rew": 4227.2634, "rew_std": 1261.4641, "Agent": "SAC"}, {"env_step": 590000, "rew": 4496.8565, "rew_std": 1100.5603, "Agent": "SAC"}, {"env_step": 595000, "rew": 4897.1523, "rew_std": 673.0572, "Agent": "SAC"}, {"env_step": 600000, "rew": 4824.998, "rew_std": 1298.6373, "Agent": "SAC"}, {"env_step": 605000, "rew": 4338.918, "rew_std": 1322.4879, "Agent": "SAC"}, {"env_step": 610000, "rew": 4937.4641, "rew_std": 694.7557, "Agent": "SAC"}, {"env_step": 615000, "rew": 5122.7475, "rew_std": 560.8034, "Agent": "SAC"}, {"env_step": 620000, "rew": 4848.3673, "rew_std": 1080.9785, "Agent": "SAC"}, {"env_step": 625000, "rew": 5232.965, "rew_std": 625.975, "Agent": "SAC"}, {"env_step": 630000, "rew": 5002.0817, "rew_std": 737.5698, "Agent": "SAC"}, {"env_step": 635000, "rew": 5374.9484, "rew_std": 216.2208, "Agent": "SAC"}, {"env_step": 640000, "rew": 5125.9609, "rew_std": 1028.8434, "Agent": "SAC"}, {"env_step": 645000, "rew": 4931.8062, "rew_std": 1508.0666, "Agent": "SAC"}, {"env_step": 650000, "rew": 4572.6578, "rew_std": 1386.8231, "Agent": "SAC"}, {"env_step": 655000, "rew": 4731.1344, "rew_std": 1108.9666, "Agent": "SAC"}, {"env_step": 660000, "rew": 4734.3651, "rew_std": 1294.0055, "Agent": "SAC"}, {"env_step": 665000, "rew": 4941.1906, "rew_std": 622.0798, "Agent": "SAC"}, {"env_step": 670000, "rew": 4804.5048, "rew_std": 890.0309, "Agent": "SAC"}, {"env_step": 675000, "rew": 4684.468, "rew_std": 821.0354, "Agent": "SAC"}, {"env_step": 680000, "rew": 5363.6274, "rew_std": 164.0225, "Agent": "SAC"}, {"env_step": 685000, "rew": 5001.633, "rew_std": 1051.6927, "Agent": "SAC"}, {"env_step": 690000, "rew": 5298.3275, "rew_std": 339.2928, "Agent": "SAC"}, {"env_step": 695000, "rew": 4865.5476, "rew_std": 1470.0991, "Agent": "SAC"}, {"env_step": 700000, "rew": 4609.8391, "rew_std": 1406.8704, "Agent": "SAC"}, {"env_step": 705000, "rew": 5323.9809, "rew_std": 306.6037, "Agent": "SAC"}, {"env_step": 710000, "rew": 5128.291, "rew_std": 443.2792, "Agent": "SAC"}, {"env_step": 715000, "rew": 5192.4866, "rew_std": 402.7956, "Agent": "SAC"}, {"env_step": 720000, "rew": 5305.8674, "rew_std": 212.3237, "Agent": "SAC"}, {"env_step": 725000, "rew": 4923.7567, "rew_std": 1082.2713, "Agent": "SAC"}, {"env_step": 730000, "rew": 4893.8484, "rew_std": 1127.4641, "Agent": "SAC"}, {"env_step": 735000, "rew": 5376.7686, "rew_std": 154.4458, "Agent": "SAC"}, {"env_step": 740000, "rew": 5129.1078, "rew_std": 422.2793, "Agent": "SAC"}, {"env_step": 745000, "rew": 5254.9446, "rew_std": 345.5945, "Agent": "SAC"}, {"env_step": 750000, "rew": 5201.6401, "rew_std": 529.1572, "Agent": "SAC"}, {"env_step": 755000, "rew": 5287.1598, "rew_std": 430.0224, "Agent": "SAC"}, {"env_step": 760000, "rew": 5188.2075, "rew_std": 406.3099, "Agent": "SAC"}, {"env_step": 765000, "rew": 5321.3263, "rew_std": 380.4228, "Agent": "SAC"}, {"env_step": 770000, "rew": 5150.8102, "rew_std": 472.9078, "Agent": "SAC"}, {"env_step": 775000, "rew": 5112.3058, "rew_std": 448.8352, "Agent": "SAC"}, {"env_step": 780000, "rew": 5438.7415, "rew_std": 139.0196, "Agent": "SAC"}, {"env_step": 785000, "rew": 5424.2751, "rew_std": 219.4044, "Agent": "SAC"}, {"env_step": 790000, "rew": 5420.4271, "rew_std": 170.9598, "Agent": "SAC"}, {"env_step": 795000, "rew": 5409.7089, "rew_std": 152.7513, "Agent": "SAC"}, {"env_step": 800000, "rew": 5369.1825, "rew_std": 271.0678, "Agent": "SAC"}, {"env_step": 805000, "rew": 5061.0436, "rew_std": 772.6225, "Agent": "SAC"}, {"env_step": 810000, "rew": 4937.3571, "rew_std": 1475.3603, "Agent": "SAC"}, {"env_step": 815000, "rew": 4996.5811, "rew_std": 1230.9226, "Agent": "SAC"}, {"env_step": 820000, "rew": 5018.8375, "rew_std": 786.6848, "Agent": "SAC"}, {"env_step": 825000, "rew": 5130.769, "rew_std": 610.5639, "Agent": "SAC"}, {"env_step": 830000, "rew": 5208.4764, "rew_std": 526.7289, "Agent": "SAC"}, {"env_step": 835000, "rew": 5122.3821, "rew_std": 995.9235, "Agent": "SAC"}, {"env_step": 840000, "rew": 5104.1109, "rew_std": 827.8651, "Agent": "SAC"}, {"env_step": 845000, "rew": 5429.061, "rew_std": 141.7453, "Agent": "SAC"}, {"env_step": 850000, "rew": 5427.7533, "rew_std": 187.8448, "Agent": "SAC"}, {"env_step": 855000, "rew": 5487.2295, "rew_std": 65.2655, "Agent": "SAC"}, {"env_step": 860000, "rew": 5392.2913, "rew_std": 200.2466, "Agent": "SAC"}, {"env_step": 865000, "rew": 5458.626, "rew_std": 116.5534, "Agent": "SAC"}, {"env_step": 870000, "rew": 5341.482, "rew_std": 513.9814, "Agent": "SAC"}, {"env_step": 875000, "rew": 5148.9941, "rew_std": 933.1094, "Agent": "SAC"}, {"env_step": 880000, "rew": 5302.6708, "rew_std": 569.9856, "Agent": "SAC"}, {"env_step": 885000, "rew": 5309.8879, "rew_std": 352.142, "Agent": "SAC"}, {"env_step": 890000, "rew": 5392.5359, "rew_std": 305.3011, "Agent": "SAC"}, {"env_step": 895000, "rew": 5182.5145, "rew_std": 779.8814, "Agent": "SAC"}, {"env_step": 900000, "rew": 4960.3537, "rew_std": 1161.7267, "Agent": "SAC"}, {"env_step": 905000, "rew": 4939.2962, "rew_std": 1498.3747, "Agent": "SAC"}, {"env_step": 910000, "rew": 4885.5797, "rew_std": 1316.2522, "Agent": "SAC"}, {"env_step": 915000, "rew": 5341.225, "rew_std": 407.8162, "Agent": "SAC"}, {"env_step": 920000, "rew": 5306.9093, "rew_std": 475.8455, "Agent": "SAC"}, {"env_step": 925000, "rew": 5105.3437, "rew_std": 579.3931, "Agent": "SAC"}, {"env_step": 930000, "rew": 5402.4238, "rew_std": 152.8403, "Agent": "SAC"}, {"env_step": 935000, "rew": 5371.9716, "rew_std": 303.9729, "Agent": "SAC"}, {"env_step": 940000, "rew": 4486.8878, "rew_std": 1637.3413, "Agent": "SAC"}, {"env_step": 945000, "rew": 4823.5619, "rew_std": 1488.3126, "Agent": "SAC"}, {"env_step": 950000, "rew": 4822.0419, "rew_std": 1248.3136, "Agent": "SAC"}, {"env_step": 955000, "rew": 5447.5403, "rew_std": 165.6376, "Agent": "SAC"}, {"env_step": 960000, "rew": 5380.7014, "rew_std": 231.9971, "Agent": "SAC"}, {"env_step": 965000, "rew": 4889.1171, "rew_std": 894.6478, "Agent": "SAC"}, {"env_step": 970000, "rew": 5488.5109, "rew_std": 81.1671, "Agent": "SAC"}, {"env_step": 975000, "rew": 5480.2766, "rew_std": 153.2717, "Agent": "SAC"}, {"env_step": 980000, "rew": 5278.1923, "rew_std": 386.9493, "Agent": "SAC"}, {"env_step": 985000, "rew": 4943.0578, "rew_std": 1237.4507, "Agent": "SAC"}, {"env_step": 990000, "rew": 5409.148, "rew_std": 380.3438, "Agent": "SAC"}, {"env_step": 995000, "rew": 5483.5326, "rew_std": 103.0683, "Agent": "SAC"}, {"env_step": 1000000, "rew": 5415.5553, "rew_std": 214.0002, "Agent": "SAC"}, {"env_step": 1005000, "rew": 4959.1109, "rew_std": 1027.8864, "Agent": "SAC"}, {"env_step": 1010000, "rew": 5287.968, "rew_std": 535.2375, "Agent": "SAC"}, {"env_step": 1015000, "rew": 5500.3241, "rew_std": 233.4036, "Agent": "SAC"}, {"env_step": 1020000, "rew": 5212.1734, "rew_std": 952.0981, "Agent": "SAC"}, {"env_step": 1025000, "rew": 4906.8683, "rew_std": 966.5824, "Agent": "SAC"}, {"env_step": 1030000, "rew": 5261.8926, "rew_std": 551.9458, "Agent": "SAC"}, {"env_step": 1035000, "rew": 4983.7048, "rew_std": 1141.9197, "Agent": "SAC"}, {"env_step": 1040000, "rew": 5295.4149, "rew_std": 447.8563, "Agent": "SAC"}, {"env_step": 1045000, "rew": 4679.6166, "rew_std": 1609.6388, "Agent": "SAC"}, {"env_step": 1050000, "rew": 4691.6723, "rew_std": 1552.6393, "Agent": "SAC"}, {"env_step": 1055000, "rew": 5203.9302, "rew_std": 994.1317, "Agent": "SAC"}, {"env_step": 1060000, "rew": 5220.7184, "rew_std": 593.7466, "Agent": "SAC"}, {"env_step": 1065000, "rew": 5501.3539, "rew_std": 181.2919, "Agent": "SAC"}, {"env_step": 1070000, "rew": 5522.3152, "rew_std": 170.9723, "Agent": "SAC"}, {"env_step": 1075000, "rew": 5227.2177, "rew_std": 999.9346, "Agent": "SAC"}, {"env_step": 1080000, "rew": 5061.4714, "rew_std": 846.1303, "Agent": "SAC"}, {"env_step": 1085000, "rew": 5546.8159, "rew_std": 84.1642, "Agent": "SAC"}, {"env_step": 1090000, "rew": 5231.8019, "rew_std": 575.674, "Agent": "SAC"}, {"env_step": 1095000, "rew": 5367.6769, "rew_std": 420.0235, "Agent": "SAC"}, {"env_step": 1100000, "rew": 5506.1081, "rew_std": 179.8471, "Agent": "SAC"}, {"env_step": 1105000, "rew": 5601.4685, "rew_std": 68.1431, "Agent": "SAC"}, {"env_step": 1110000, "rew": 5408.7534, "rew_std": 325.9523, "Agent": "SAC"}, {"env_step": 1115000, "rew": 5535.2153, "rew_std": 186.8377, "Agent": "SAC"}, {"env_step": 1120000, "rew": 5532.6407, "rew_std": 205.192, "Agent": "SAC"}, {"env_step": 1125000, "rew": 5552.3668, "rew_std": 77.1201, "Agent": "SAC"}, {"env_step": 1130000, "rew": 5129.3217, "rew_std": 1263.6823, "Agent": "SAC"}, {"env_step": 1135000, "rew": 5500.5953, "rew_std": 187.8342, "Agent": "SAC"}, {"env_step": 1140000, "rew": 5503.6788, "rew_std": 275.1286, "Agent": "SAC"}, {"env_step": 1145000, "rew": 4995.5923, "rew_std": 1255.58, "Agent": "SAC"}, {"env_step": 1150000, "rew": 5248.8098, "rew_std": 751.2548, "Agent": "SAC"}, {"env_step": 1155000, "rew": 5435.8424, "rew_std": 209.4219, "Agent": "SAC"}, {"env_step": 1160000, "rew": 5527.5078, "rew_std": 200.7482, "Agent": "SAC"}, {"env_step": 1165000, "rew": 5550.173, "rew_std": 65.9799, "Agent": "SAC"}, {"env_step": 1170000, "rew": 5443.7728, "rew_std": 418.8417, "Agent": "SAC"}, {"env_step": 1175000, "rew": 5464.0429, "rew_std": 348.5082, "Agent": "SAC"}, {"env_step": 1180000, "rew": 5568.4337, "rew_std": 82.9834, "Agent": "SAC"}, {"env_step": 1185000, "rew": 5524.763, "rew_std": 171.4394, "Agent": "SAC"}, {"env_step": 1190000, "rew": 5212.8234, "rew_std": 1062.9118, "Agent": "SAC"}, {"env_step": 1195000, "rew": 5102.0455, "rew_std": 1028.9742, "Agent": "SAC"}, {"env_step": 1200000, "rew": 5481.4111, "rew_std": 230.1548, "Agent": "SAC"}, {"env_step": 1205000, "rew": 4889.8177, "rew_std": 1342.0486, "Agent": "SAC"}, {"env_step": 1210000, "rew": 5501.9494, "rew_std": 315.0388, "Agent": "SAC"}, {"env_step": 1215000, "rew": 5507.2537, "rew_std": 196.4056, "Agent": "SAC"}, {"env_step": 1220000, "rew": 5481.4773, "rew_std": 304.5546, "Agent": "SAC"}, {"env_step": 1225000, "rew": 5331.5667, "rew_std": 802.8777, "Agent": "SAC"}, {"env_step": 1230000, "rew": 5621.8863, "rew_std": 65.417, "Agent": "SAC"}, {"env_step": 1235000, "rew": 5310.1082, "rew_std": 560.1617, "Agent": "SAC"}, {"env_step": 1240000, "rew": 5518.8848, "rew_std": 221.8269, "Agent": "SAC"}, {"env_step": 1245000, "rew": 5296.2923, "rew_std": 998.7259, "Agent": "SAC"}, {"env_step": 1250000, "rew": 5378.8669, "rew_std": 392.2601, "Agent": "SAC"}, {"env_step": 0, "rew": 177.3211, "rew_std": 77.5947, "Agent": "DDPG"}, {"env_step": 5000, "rew": 107.4722, "rew_std": 49.5394, "Agent": "DDPG"}, {"env_step": 10000, "rew": 107.419, "rew_std": 48.3793, "Agent": "DDPG"}, {"env_step": 15000, "rew": 107.8837, "rew_std": 48.1536, "Agent": "DDPG"}, {"env_step": 20000, "rew": 108.3848, "rew_std": 49.3729, "Agent": "DDPG"}, {"env_step": 25000, "rew": 109.1298, "rew_std": 50.4306, "Agent": "DDPG"}, {"env_step": 30000, "rew": 107.7117, "rew_std": 48.689, "Agent": "DDPG"}, {"env_step": 35000, "rew": 108.4788, "rew_std": 50.029, "Agent": "DDPG"}, {"env_step": 40000, "rew": 106.8252, "rew_std": 47.411, "Agent": "DDPG"}, {"env_step": 45000, "rew": 107.2152, "rew_std": 49.5615, "Agent": "DDPG"}, {"env_step": 50000, "rew": 108.1668, "rew_std": 49.1587, "Agent": "DDPG"}, {"env_step": 55000, "rew": 107.585, "rew_std": 48.4752, "Agent": "DDPG"}, {"env_step": 60000, "rew": 107.9617, "rew_std": 48.7848, "Agent": "DDPG"}, {"env_step": 65000, "rew": 108.5746, "rew_std": 49.0814, "Agent": "DDPG"}, {"env_step": 70000, "rew": 108.5917, "rew_std": 49.2006, "Agent": "DDPG"}, {"env_step": 75000, "rew": 108.2389, "rew_std": 48.5646, "Agent": "DDPG"}, {"env_step": 80000, "rew": 106.9407, "rew_std": 47.9692, "Agent": "DDPG"}, {"env_step": 85000, "rew": 107.1787, "rew_std": 48.6623, "Agent": "DDPG"}, {"env_step": 90000, "rew": 108.0858, "rew_std": 49.8083, "Agent": "DDPG"}, {"env_step": 95000, "rew": 107.9608, "rew_std": 49.5357, "Agent": "DDPG"}, {"env_step": 100000, "rew": 107.8685, "rew_std": 49.102, "Agent": "DDPG"}, {"env_step": 105000, "rew": 108.3229, "rew_std": 49.2733, "Agent": "DDPG"}, {"env_step": 110000, "rew": 108.9463, "rew_std": 49.2226, "Agent": "DDPG"}, {"env_step": 115000, "rew": 108.6805, "rew_std": 49.843, "Agent": "DDPG"}, {"env_step": 120000, "rew": 107.3607, "rew_std": 48.9263, "Agent": "DDPG"}, {"env_step": 125000, "rew": 109.0114, "rew_std": 50.1977, "Agent": "DDPG"}, {"env_step": 130000, "rew": 108.7159, "rew_std": 50.2207, "Agent": "DDPG"}, {"env_step": 135000, "rew": 107.8893, "rew_std": 49.0612, "Agent": "DDPG"}, {"env_step": 140000, "rew": 109.5944, "rew_std": 51.1663, "Agent": "DDPG"}, {"env_step": 145000, "rew": 109.2089, "rew_std": 50.4391, "Agent": "DDPG"}, {"env_step": 150000, "rew": 108.8583, "rew_std": 49.8412, "Agent": "DDPG"}, {"env_step": 155000, "rew": 108.5092, "rew_std": 49.0027, "Agent": "DDPG"}, {"env_step": 160000, "rew": 107.0541, "rew_std": 47.7677, "Agent": "DDPG"}, {"env_step": 165000, "rew": 108.3194, "rew_std": 49.2564, "Agent": "DDPG"}, {"env_step": 170000, "rew": 108.7133, "rew_std": 50.4113, "Agent": "DDPG"}, {"env_step": 175000, "rew": 108.8514, "rew_std": 50.5438, "Agent": "DDPG"}, {"env_step": 180000, "rew": 108.6988, "rew_std": 49.5952, "Agent": "DDPG"}, {"env_step": 185000, "rew": 107.0648, "rew_std": 47.8116, "Agent": "DDPG"}, {"env_step": 190000, "rew": 107.8391, "rew_std": 48.4472, "Agent": "DDPG"}, {"env_step": 195000, "rew": 108.5129, "rew_std": 49.9494, "Agent": "DDPG"}, {"env_step": 200000, "rew": 108.1958, "rew_std": 50.1158, "Agent": "DDPG"}, {"env_step": 205000, "rew": 108.3988, "rew_std": 49.3048, "Agent": "DDPG"}, {"env_step": 210000, "rew": 107.4057, "rew_std": 48.2847, "Agent": "DDPG"}, {"env_step": 215000, "rew": 107.1662, "rew_std": 48.5738, "Agent": "DDPG"}, {"env_step": 220000, "rew": 108.6186, "rew_std": 49.7125, "Agent": "DDPG"}, {"env_step": 225000, "rew": 108.11, "rew_std": 48.6119, "Agent": "DDPG"}, {"env_step": 230000, "rew": 107.9753, "rew_std": 49.3788, "Agent": "DDPG"}, {"env_step": 235000, "rew": 108.1134, "rew_std": 48.8535, "Agent": "DDPG"}, {"env_step": 240000, "rew": 108.3508, "rew_std": 49.4337, "Agent": "DDPG"}, {"env_step": 245000, "rew": 107.1296, "rew_std": 47.9657, "Agent": "DDPG"}, {"env_step": 250000, "rew": 109.8651, "rew_std": 51.256, "Agent": "DDPG"}, {"env_step": 255000, "rew": 107.0648, "rew_std": 48.2469, "Agent": "DDPG"}, {"env_step": 260000, "rew": 107.1931, "rew_std": 48.5244, "Agent": "DDPG"}, {"env_step": 265000, "rew": 108.8675, "rew_std": 50.6869, "Agent": "DDPG"}, {"env_step": 270000, "rew": 107.7211, "rew_std": 48.9009, "Agent": "DDPG"}, {"env_step": 275000, "rew": 106.8022, "rew_std": 47.7847, "Agent": "DDPG"}, {"env_step": 280000, "rew": 108.1544, "rew_std": 48.4929, "Agent": "DDPG"}, {"env_step": 285000, "rew": 107.3888, "rew_std": 48.73, "Agent": "DDPG"}, {"env_step": 290000, "rew": 108.1, "rew_std": 49.7088, "Agent": "DDPG"}, {"env_step": 295000, "rew": 107.7946, "rew_std": 48.7758, "Agent": "DDPG"}, {"env_step": 300000, "rew": 108.2477, "rew_std": 49.7653, "Agent": "DDPG"}, {"env_step": 305000, "rew": 106.3818, "rew_std": 48.0367, "Agent": "DDPG"}, {"env_step": 310000, "rew": 108.6188, "rew_std": 49.6778, "Agent": "DDPG"}, {"env_step": 315000, "rew": 108.7012, "rew_std": 49.2731, "Agent": "DDPG"}, {"env_step": 320000, "rew": 108.9132, "rew_std": 50.1521, "Agent": "DDPG"}, {"env_step": 325000, "rew": 109.3433, "rew_std": 50.3347, "Agent": "DDPG"}, {"env_step": 330000, "rew": 106.6812, "rew_std": 48.186, "Agent": "DDPG"}, {"env_step": 335000, "rew": 107.5278, "rew_std": 48.4076, "Agent": "DDPG"}, {"env_step": 340000, "rew": 109.3871, "rew_std": 50.6109, "Agent": "DDPG"}, {"env_step": 345000, "rew": 107.0201, "rew_std": 47.9292, "Agent": "DDPG"}, {"env_step": 350000, "rew": 108.046, "rew_std": 48.6814, "Agent": "DDPG"}, {"env_step": 355000, "rew": 108.6397, "rew_std": 50.246, "Agent": "DDPG"}, {"env_step": 360000, "rew": 107.8538, "rew_std": 49.0871, "Agent": "DDPG"}, {"env_step": 365000, "rew": 108.441, "rew_std": 49.4998, "Agent": "DDPG"}, {"env_step": 370000, "rew": 108.9821, "rew_std": 49.8238, "Agent": "DDPG"}, {"env_step": 375000, "rew": 107.8918, "rew_std": 49.6536, "Agent": "DDPG"}, {"env_step": 380000, "rew": 107.9529, "rew_std": 49.4383, "Agent": "DDPG"}, {"env_step": 385000, "rew": 107.6085, "rew_std": 49.2473, "Agent": "DDPG"}, {"env_step": 390000, "rew": 108.3302, "rew_std": 49.122, "Agent": "DDPG"}, {"env_step": 395000, "rew": 109.8158, "rew_std": 51.3892, "Agent": "DDPG"}, {"env_step": 400000, "rew": 107.381, "rew_std": 48.4753, "Agent": "DDPG"}, {"env_step": 405000, "rew": 108.6191, "rew_std": 49.4195, "Agent": "DDPG"}, {"env_step": 410000, "rew": 107.41, "rew_std": 48.837, "Agent": "DDPG"}, {"env_step": 415000, "rew": 108.4352, "rew_std": 49.2737, "Agent": "DDPG"}, {"env_step": 420000, "rew": 107.9026, "rew_std": 48.534, "Agent": "DDPG"}, {"env_step": 425000, "rew": 108.478, "rew_std": 48.7426, "Agent": "DDPG"}, {"env_step": 430000, "rew": 107.1057, "rew_std": 47.5283, "Agent": "DDPG"}, {"env_step": 435000, "rew": 107.6836, "rew_std": 48.4309, "Agent": "DDPG"}, {"env_step": 440000, "rew": 108.1322, "rew_std": 48.2125, "Agent": "DDPG"}, {"env_step": 445000, "rew": 107.9857, "rew_std": 49.0165, "Agent": "DDPG"}, {"env_step": 450000, "rew": 109.4034, "rew_std": 51.0131, "Agent": "DDPG"}, {"env_step": 455000, "rew": 107.6908, "rew_std": 48.9333, "Agent": "DDPG"}, {"env_step": 460000, "rew": 107.0666, "rew_std": 47.51, "Agent": "DDPG"}, {"env_step": 465000, "rew": 109.4376, "rew_std": 50.1125, "Agent": "DDPG"}, {"env_step": 470000, "rew": 108.6718, "rew_std": 49.2696, "Agent": "DDPG"}, {"env_step": 475000, "rew": 107.5808, "rew_std": 49.0431, "Agent": "DDPG"}, {"env_step": 480000, "rew": 107.2121, "rew_std": 48.7748, "Agent": "DDPG"}, {"env_step": 485000, "rew": 107.1728, "rew_std": 48.2687, "Agent": "DDPG"}, {"env_step": 490000, "rew": 108.0973, "rew_std": 49.3657, "Agent": "DDPG"}, {"env_step": 495000, "rew": 107.7327, "rew_std": 48.7304, "Agent": "DDPG"}, {"env_step": 500000, "rew": 108.8858, "rew_std": 49.8917, "Agent": "DDPG"}, {"env_step": 505000, "rew": 108.0974, "rew_std": 49.7294, "Agent": "DDPG"}, {"env_step": 510000, "rew": 107.1945, "rew_std": 47.9077, "Agent": "DDPG"}, {"env_step": 515000, "rew": 108.1582, "rew_std": 48.6089, "Agent": "DDPG"}, {"env_step": 520000, "rew": 107.6266, "rew_std": 48.4478, "Agent": "DDPG"}, {"env_step": 525000, "rew": 107.2526, "rew_std": 48.5832, "Agent": "DDPG"}, {"env_step": 530000, "rew": 107.6767, "rew_std": 48.8641, "Agent": "DDPG"}, {"env_step": 535000, "rew": 108.9989, "rew_std": 49.9125, "Agent": "DDPG"}, {"env_step": 540000, "rew": 108.1414, "rew_std": 49.2883, "Agent": "DDPG"}, {"env_step": 545000, "rew": 108.7419, "rew_std": 49.8283, "Agent": "DDPG"}, {"env_step": 550000, "rew": 107.8539, "rew_std": 48.977, "Agent": "DDPG"}, {"env_step": 555000, "rew": 108.6301, "rew_std": 49.5366, "Agent": "DDPG"}, {"env_step": 560000, "rew": 105.8462, "rew_std": 47.1965, "Agent": "DDPG"}, {"env_step": 565000, "rew": 107.7476, "rew_std": 48.4578, "Agent": "DDPG"}, {"env_step": 570000, "rew": 107.11, "rew_std": 47.5214, "Agent": "DDPG"}, {"env_step": 575000, "rew": 108.8707, "rew_std": 49.2753, "Agent": "DDPG"}, {"env_step": 580000, "rew": 108.4656, "rew_std": 49.7007, "Agent": "DDPG"}, {"env_step": 585000, "rew": 107.4115, "rew_std": 48.396, "Agent": "DDPG"}, {"env_step": 590000, "rew": 108.1877, "rew_std": 48.6477, "Agent": "DDPG"}, {"env_step": 595000, "rew": 108.6227, "rew_std": 49.4947, "Agent": "DDPG"}, {"env_step": 600000, "rew": 108.855, "rew_std": 50.5568, "Agent": "DDPG"}, {"env_step": 605000, "rew": 108.0074, "rew_std": 49.2264, "Agent": "DDPG"}, {"env_step": 610000, "rew": 106.7105, "rew_std": 47.8051, "Agent": "DDPG"}, {"env_step": 615000, "rew": 107.7101, "rew_std": 48.9149, "Agent": "DDPG"}, {"env_step": 620000, "rew": 107.2254, "rew_std": 49.1405, "Agent": "DDPG"}, {"env_step": 625000, "rew": 107.5752, "rew_std": 48.2163, "Agent": "DDPG"}, {"env_step": 630000, "rew": 108.224, "rew_std": 49.7724, "Agent": "DDPG"}, {"env_step": 635000, "rew": 108.7608, "rew_std": 50.2342, "Agent": "DDPG"}, {"env_step": 640000, "rew": 107.6411, "rew_std": 48.8166, "Agent": "DDPG"}, {"env_step": 645000, "rew": 108.7065, "rew_std": 49.4438, "Agent": "DDPG"}, {"env_step": 650000, "rew": 109.1814, "rew_std": 50.5024, "Agent": "DDPG"}, {"env_step": 655000, "rew": 109.8778, "rew_std": 50.4916, "Agent": "DDPG"}, {"env_step": 660000, "rew": 108.5942, "rew_std": 49.6353, "Agent": "DDPG"}, {"env_step": 665000, "rew": 108.0445, "rew_std": 49.3306, "Agent": "DDPG"}, {"env_step": 670000, "rew": 107.5365, "rew_std": 48.3147, "Agent": "DDPG"}, {"env_step": 675000, "rew": 107.4953, "rew_std": 47.9975, "Agent": "DDPG"}, {"env_step": 680000, "rew": 108.0237, "rew_std": 48.7486, "Agent": "DDPG"}, {"env_step": 685000, "rew": 108.4675, "rew_std": 48.824, "Agent": "DDPG"}, {"env_step": 690000, "rew": 107.6245, "rew_std": 48.3674, "Agent": "DDPG"}, {"env_step": 695000, "rew": 108.8214, "rew_std": 49.8349, "Agent": "DDPG"}, {"env_step": 700000, "rew": 107.5492, "rew_std": 48.4227, "Agent": "DDPG"}, {"env_step": 705000, "rew": 107.18, "rew_std": 48.8275, "Agent": "DDPG"}, {"env_step": 710000, "rew": 108.5283, "rew_std": 49.5153, "Agent": "DDPG"}, {"env_step": 715000, "rew": 106.5051, "rew_std": 47.4221, "Agent": "DDPG"}, {"env_step": 720000, "rew": 106.9291, "rew_std": 49.2053, "Agent": "DDPG"}, {"env_step": 725000, "rew": 108.4394, "rew_std": 49.4338, "Agent": "DDPG"}, {"env_step": 730000, "rew": 107.6665, "rew_std": 49.4634, "Agent": "DDPG"}, {"env_step": 735000, "rew": 107.7959, "rew_std": 48.591, "Agent": "DDPG"}, {"env_step": 740000, "rew": 108.0495, "rew_std": 49.8033, "Agent": "DDPG"}, {"env_step": 745000, "rew": 107.4518, "rew_std": 48.6424, "Agent": "DDPG"}, {"env_step": 750000, "rew": 108.1426, "rew_std": 49.1501, "Agent": "DDPG"}, {"env_step": 755000, "rew": 108.2628, "rew_std": 50.0382, "Agent": "DDPG"}, {"env_step": 760000, "rew": 107.2477, "rew_std": 48.0286, "Agent": "DDPG"}, {"env_step": 765000, "rew": 109.0266, "rew_std": 49.8274, "Agent": "DDPG"}, {"env_step": 770000, "rew": 109.0255, "rew_std": 49.9443, "Agent": "DDPG"}, {"env_step": 775000, "rew": 108.1221, "rew_std": 49.8079, "Agent": "DDPG"}, {"env_step": 780000, "rew": 108.4238, "rew_std": 49.3669, "Agent": "DDPG"}, {"env_step": 785000, "rew": 108.115, "rew_std": 48.8943, "Agent": "DDPG"}, {"env_step": 790000, "rew": 106.6352, "rew_std": 47.2801, "Agent": "DDPG"}, {"env_step": 795000, "rew": 108.7755, "rew_std": 49.2822, "Agent": "DDPG"}, {"env_step": 800000, "rew": 107.1741, "rew_std": 48.0624, "Agent": "DDPG"}, {"env_step": 805000, "rew": 107.5149, "rew_std": 48.7057, "Agent": "DDPG"}, {"env_step": 810000, "rew": 108.7222, "rew_std": 50.0176, "Agent": "DDPG"}, {"env_step": 815000, "rew": 107.5641, "rew_std": 48.7545, "Agent": "DDPG"}, {"env_step": 820000, "rew": 107.6578, "rew_std": 49.0399, "Agent": "DDPG"}, {"env_step": 825000, "rew": 108.2342, "rew_std": 49.4261, "Agent": "DDPG"}, {"env_step": 830000, "rew": 107.9824, "rew_std": 49.6815, "Agent": "DDPG"}, {"env_step": 835000, "rew": 108.5656, "rew_std": 49.6647, "Agent": "DDPG"}, {"env_step": 840000, "rew": 107.2489, "rew_std": 48.1444, "Agent": "DDPG"}, {"env_step": 845000, "rew": 108.5607, "rew_std": 48.6598, "Agent": "DDPG"}, {"env_step": 850000, "rew": 108.9987, "rew_std": 49.9275, "Agent": "DDPG"}, {"env_step": 855000, "rew": 108.4097, "rew_std": 49.2914, "Agent": "DDPG"}, {"env_step": 860000, "rew": 108.8319, "rew_std": 50.0234, "Agent": "DDPG"}, {"env_step": 865000, "rew": 107.2195, "rew_std": 48.4018, "Agent": "DDPG"}, {"env_step": 870000, "rew": 107.3554, "rew_std": 48.6998, "Agent": "DDPG"}, {"env_step": 875000, "rew": 108.5066, "rew_std": 49.5775, "Agent": "DDPG"}, {"env_step": 880000, "rew": 107.5358, "rew_std": 48.8969, "Agent": "DDPG"}, {"env_step": 885000, "rew": 107.4236, "rew_std": 48.9267, "Agent": "DDPG"}, {"env_step": 890000, "rew": 109.3625, "rew_std": 50.3215, "Agent": "DDPG"}, {"env_step": 895000, "rew": 107.8831, "rew_std": 48.6233, "Agent": "DDPG"}, {"env_step": 900000, "rew": 109.3476, "rew_std": 50.3894, "Agent": "DDPG"}, {"env_step": 905000, "rew": 107.8047, "rew_std": 49.2224, "Agent": "DDPG"}, {"env_step": 910000, "rew": 107.7403, "rew_std": 48.9944, "Agent": "DDPG"}, {"env_step": 915000, "rew": 107.7707, "rew_std": 48.8078, "Agent": "DDPG"}, {"env_step": 920000, "rew": 108.6377, "rew_std": 49.3496, "Agent": "DDPG"}, {"env_step": 925000, "rew": 108.3392, "rew_std": 48.886, "Agent": "DDPG"}, {"env_step": 930000, "rew": 107.7412, "rew_std": 49.0511, "Agent": "DDPG"}, {"env_step": 935000, "rew": 108.4689, "rew_std": 49.3729, "Agent": "DDPG"}, {"env_step": 940000, "rew": 108.3622, "rew_std": 48.773, "Agent": "DDPG"}, {"env_step": 945000, "rew": 108.3962, "rew_std": 49.1464, "Agent": "DDPG"}, {"env_step": 950000, "rew": 107.2917, "rew_std": 48.0124, "Agent": "DDPG"}, {"env_step": 955000, "rew": 107.329, "rew_std": 48.7632, "Agent": "DDPG"}, {"env_step": 960000, "rew": 107.8866, "rew_std": 49.1713, "Agent": "DDPG"}, {"env_step": 965000, "rew": 107.5759, "rew_std": 48.6815, "Agent": "DDPG"}, {"env_step": 970000, "rew": 108.133, "rew_std": 49.4281, "Agent": "DDPG"}, {"env_step": 975000, "rew": 107.6389, "rew_std": 48.2696, "Agent": "DDPG"}, {"env_step": 980000, "rew": 107.4483, "rew_std": 48.6085, "Agent": "DDPG"}, {"env_step": 985000, "rew": 108.0189, "rew_std": 48.8417, "Agent": "DDPG"}, {"env_step": 990000, "rew": 109.3234, "rew_std": 50.5991, "Agent": "DDPG"}, {"env_step": 995000, "rew": 109.0151, "rew_std": 50.4905, "Agent": "DDPG"}, {"env_step": 1000000, "rew": 108.814, "rew_std": 49.8831, "Agent": "DDPG"}, {"env_step": 1005000, "rew": 108.3801, "rew_std": 49.4534, "Agent": "DDPG"}, {"env_step": 1010000, "rew": 108.5171, "rew_std": 49.7201, "Agent": "DDPG"}, {"env_step": 1015000, "rew": 108.6333, "rew_std": 49.3683, "Agent": "DDPG"}, {"env_step": 1020000, "rew": 107.8037, "rew_std": 48.8344, "Agent": "DDPG"}, {"env_step": 1025000, "rew": 109.6875, "rew_std": 51.1063, "Agent": "DDPG"}, {"env_step": 1030000, "rew": 107.952, "rew_std": 49.3687, "Agent": "DDPG"}, {"env_step": 1035000, "rew": 108.3813, "rew_std": 48.8169, "Agent": "DDPG"}, {"env_step": 1040000, "rew": 108.3889, "rew_std": 49.3835, "Agent": "DDPG"}, {"env_step": 1045000, "rew": 108.832, "rew_std": 50.5686, "Agent": "DDPG"}, {"env_step": 1050000, "rew": 108.2503, "rew_std": 49.4035, "Agent": "DDPG"}, {"env_step": 1055000, "rew": 107.518, "rew_std": 48.8938, "Agent": "DDPG"}, {"env_step": 1060000, "rew": 108.592, "rew_std": 50.0621, "Agent": "DDPG"}, {"env_step": 1065000, "rew": 108.156, "rew_std": 48.9436, "Agent": "DDPG"}, {"env_step": 1070000, "rew": 107.79, "rew_std": 49.1456, "Agent": "DDPG"}, {"env_step": 1075000, "rew": 107.3237, "rew_std": 48.2445, "Agent": "DDPG"}, {"env_step": 1080000, "rew": 108.0468, "rew_std": 49.3038, "Agent": "DDPG"}, {"env_step": 1085000, "rew": 107.2851, "rew_std": 48.7523, "Agent": "DDPG"}, {"env_step": 1090000, "rew": 109.5256, "rew_std": 50.6178, "Agent": "DDPG"}, {"env_step": 1095000, "rew": 107.6415, "rew_std": 49.1169, "Agent": "DDPG"}, {"env_step": 1100000, "rew": 108.1626, "rew_std": 49.8981, "Agent": "DDPG"}, {"env_step": 1105000, "rew": 106.2928, "rew_std": 47.0271, "Agent": "DDPG"}, {"env_step": 1110000, "rew": 107.5527, "rew_std": 49.2958, "Agent": "DDPG"}, {"env_step": 1115000, "rew": 108.6973, "rew_std": 49.8643, "Agent": "DDPG"}, {"env_step": 1120000, "rew": 108.662, "rew_std": 49.4262, "Agent": "DDPG"}, {"env_step": 1125000, "rew": 106.7754, "rew_std": 47.1026, "Agent": "DDPG"}, {"env_step": 1130000, "rew": 107.6413, "rew_std": 48.6138, "Agent": "DDPG"}, {"env_step": 1135000, "rew": 108.61, "rew_std": 49.4814, "Agent": "DDPG"}, {"env_step": 1140000, "rew": 108.4586, "rew_std": 49.4734, "Agent": "DDPG"}, {"env_step": 1145000, "rew": 108.5004, "rew_std": 49.183, "Agent": "DDPG"}, {"env_step": 1150000, "rew": 107.3857, "rew_std": 48.3934, "Agent": "DDPG"}, {"env_step": 1155000, "rew": 107.8232, "rew_std": 49.21, "Agent": "DDPG"}, {"env_step": 1160000, "rew": 107.4314, "rew_std": 48.6418, "Agent": "DDPG"}, {"env_step": 1165000, "rew": 108.6097, "rew_std": 50.0509, "Agent": "DDPG"}, {"env_step": 1170000, "rew": 106.9657, "rew_std": 48.2271, "Agent": "DDPG"}, {"env_step": 1175000, "rew": 107.1207, "rew_std": 47.8233, "Agent": "DDPG"}, {"env_step": 1180000, "rew": 108.4552, "rew_std": 48.7664, "Agent": "DDPG"}, {"env_step": 1185000, "rew": 107.2896, "rew_std": 47.9247, "Agent": "DDPG"}, {"env_step": 1190000, "rew": 109.0348, "rew_std": 50.3553, "Agent": "DDPG"}, {"env_step": 1195000, "rew": 107.02, "rew_std": 47.8831, "Agent": "DDPG"}, {"env_step": 1200000, "rew": 107.8539, "rew_std": 48.963, "Agent": "DDPG"}, {"env_step": 1205000, "rew": 109.2758, "rew_std": 50.1332, "Agent": "DDPG"}, {"env_step": 1210000, "rew": 108.532, "rew_std": 49.2018, "Agent": "DDPG"}, {"env_step": 1215000, "rew": 107.9433, "rew_std": 49.1756, "Agent": "DDPG"}, {"env_step": 1220000, "rew": 108.429, "rew_std": 50.3263, "Agent": "DDPG"}, {"env_step": 1225000, "rew": 108.4288, "rew_std": 49.6, "Agent": "DDPG"}, {"env_step": 1230000, "rew": 108.0104, "rew_std": 49.69, "Agent": "DDPG"}, {"env_step": 1235000, "rew": 106.8428, "rew_std": 48.5726, "Agent": "DDPG"}, {"env_step": 1240000, "rew": 107.897, "rew_std": 49.5459, "Agent": "DDPG"}, {"env_step": 1245000, "rew": 107.7359, "rew_std": 48.8844, "Agent": "DDPG"}, {"env_step": 1250000, "rew": 107.0363, "rew_std": 48.3924, "Agent": "DDPG"}, {"env_step": 5000, "rew": 235.9497, "rew_std": 126.1434, "Agent": "TD3"}, {"env_step": 10000, "rew": 164.07, "rew_std": 81.7212, "Agent": "TD3"}, {"env_step": 15000, "rew": 211.2634, "rew_std": 79.2344, "Agent": "TD3"}, {"env_step": 20000, "rew": 282.4783, "rew_std": 93.0359, "Agent": "TD3"}, {"env_step": 25000, "rew": 318.5609, "rew_std": 73.6385, "Agent": "TD3"}, {"env_step": 30000, "rew": 291.8901, "rew_std": 40.7493, "Agent": "TD3"}, {"env_step": 35000, "rew": 313.7815, "rew_std": 73.0841, "Agent": "TD3"}, {"env_step": 40000, "rew": 337.9159, "rew_std": 57.573, "Agent": "TD3"}, {"env_step": 45000, "rew": 368.3768, "rew_std": 92.3044, "Agent": "TD3"}, {"env_step": 50000, "rew": 344.416, "rew_std": 76.0211, "Agent": "TD3"}, {"env_step": 55000, "rew": 429.7109, "rew_std": 82.0808, "Agent": "TD3"}, {"env_step": 60000, "rew": 395.007, "rew_std": 63.4624, "Agent": "TD3"}, {"env_step": 65000, "rew": 415.4698, "rew_std": 62.3147, "Agent": "TD3"}, {"env_step": 70000, "rew": 459.2035, "rew_std": 48.2942, "Agent": "TD3"}, {"env_step": 75000, "rew": 452.3541, "rew_std": 65.0611, "Agent": "TD3"}, {"env_step": 80000, "rew": 461.8969, "rew_std": 63.6509, "Agent": "TD3"}, {"env_step": 85000, "rew": 476.0453, "rew_std": 71.275, "Agent": "TD3"}, {"env_step": 90000, "rew": 500.811, "rew_std": 84.8932, "Agent": "TD3"}, {"env_step": 95000, "rew": 485.3997, "rew_std": 80.3051, "Agent": "TD3"}, {"env_step": 100000, "rew": 503.4416, "rew_std": 42.9756, "Agent": "TD3"}, {"env_step": 105000, "rew": 527.5804, "rew_std": 77.3744, "Agent": "TD3"}, {"env_step": 110000, "rew": 544.3686, "rew_std": 55.0087, "Agent": "TD3"}, {"env_step": 115000, "rew": 553.5231, "rew_std": 50.6329, "Agent": "TD3"}, {"env_step": 120000, "rew": 574.4474, "rew_std": 75.047, "Agent": "TD3"}, {"env_step": 125000, "rew": 559.3873, "rew_std": 86.2582, "Agent": "TD3"}, {"env_step": 130000, "rew": 570.9049, "rew_std": 59.5154, "Agent": "TD3"}, {"env_step": 135000, "rew": 580.3405, "rew_std": 50.9925, "Agent": "TD3"}, {"env_step": 140000, "rew": 606.0723, "rew_std": 62.0672, "Agent": "TD3"}, {"env_step": 145000, "rew": 605.5913, "rew_std": 31.7131, "Agent": "TD3"}, {"env_step": 150000, "rew": 625.5167, "rew_std": 66.0565, "Agent": "TD3"}, {"env_step": 155000, "rew": 652.0839, "rew_std": 133.1029, "Agent": "TD3"}, {"env_step": 160000, "rew": 677.93, "rew_std": 73.4751, "Agent": "TD3"}, {"env_step": 165000, "rew": 660.96, "rew_std": 107.0351, "Agent": "TD3"}, {"env_step": 170000, "rew": 667.0291, "rew_std": 57.4566, "Agent": "TD3"}, {"env_step": 175000, "rew": 699.7508, "rew_std": 86.6757, "Agent": "TD3"}, {"env_step": 180000, "rew": 724.8383, "rew_std": 124.4392, "Agent": "TD3"}, {"env_step": 185000, "rew": 792.9929, "rew_std": 102.3493, "Agent": "TD3"}, {"env_step": 190000, "rew": 801.6827, "rew_std": 176.7923, "Agent": "TD3"}, {"env_step": 195000, "rew": 860.7451, "rew_std": 123.1047, "Agent": "TD3"}, {"env_step": 200000, "rew": 831.1765, "rew_std": 115.7513, "Agent": "TD3"}, {"env_step": 205000, "rew": 891.3844, "rew_std": 153.8025, "Agent": "TD3"}, {"env_step": 210000, "rew": 920.1051, "rew_std": 138.4675, "Agent": "TD3"}, {"env_step": 215000, "rew": 968.2489, "rew_std": 206.803, "Agent": "TD3"}, {"env_step": 220000, "rew": 987.881, "rew_std": 128.4658, "Agent": "TD3"}, {"env_step": 225000, "rew": 1061.9422, "rew_std": 223.8467, "Agent": "TD3"}, {"env_step": 230000, "rew": 1259.2323, "rew_std": 497.5445, "Agent": "TD3"}, {"env_step": 235000, "rew": 1096.4967, "rew_std": 141.7399, "Agent": "TD3"}, {"env_step": 240000, "rew": 1311.5082, "rew_std": 429.5385, "Agent": "TD3"}, {"env_step": 245000, "rew": 1311.3561, "rew_std": 450.7312, "Agent": "TD3"}, {"env_step": 250000, "rew": 1557.6187, "rew_std": 1001.0859, "Agent": "TD3"}, {"env_step": 255000, "rew": 1492.0918, "rew_std": 536.5407, "Agent": "TD3"}, {"env_step": 260000, "rew": 1478.4514, "rew_std": 597.8008, "Agent": "TD3"}, {"env_step": 265000, "rew": 1759.7188, "rew_std": 679.3698, "Agent": "TD3"}, {"env_step": 270000, "rew": 1536.9493, "rew_std": 357.449, "Agent": "TD3"}, {"env_step": 275000, "rew": 1667.6531, "rew_std": 376.0051, "Agent": "TD3"}, {"env_step": 280000, "rew": 1751.9256, "rew_std": 554.9098, "Agent": "TD3"}, {"env_step": 285000, "rew": 2053.3891, "rew_std": 597.2069, "Agent": "TD3"}, {"env_step": 290000, "rew": 2180.2304, "rew_std": 747.0793, "Agent": "TD3"}, {"env_step": 295000, "rew": 2150.2635, "rew_std": 542.9699, "Agent": "TD3"}, {"env_step": 300000, "rew": 2311.3341, "rew_std": 776.4552, "Agent": "TD3"}, {"env_step": 305000, "rew": 2605.4033, "rew_std": 969.016, "Agent": "TD3"}, {"env_step": 310000, "rew": 2496.6585, "rew_std": 490.1734, "Agent": "TD3"}, {"env_step": 315000, "rew": 2636.3198, "rew_std": 815.1771, "Agent": "TD3"}, {"env_step": 320000, "rew": 2553.8276, "rew_std": 1048.6062, "Agent": "TD3"}, {"env_step": 325000, "rew": 2962.0407, "rew_std": 1075.7931, "Agent": "TD3"}, {"env_step": 330000, "rew": 2935.3863, "rew_std": 1000.7385, "Agent": "TD3"}, {"env_step": 335000, "rew": 2921.3217, "rew_std": 774.101, "Agent": "TD3"}, {"env_step": 340000, "rew": 3469.4737, "rew_std": 1022.4847, "Agent": "TD3"}, {"env_step": 345000, "rew": 3185.8696, "rew_std": 733.3435, "Agent": "TD3"}, {"env_step": 350000, "rew": 3575.7222, "rew_std": 921.7069, "Agent": "TD3"}, {"env_step": 355000, "rew": 3627.7872, "rew_std": 1142.049, "Agent": "TD3"}, {"env_step": 360000, "rew": 3227.9706, "rew_std": 711.5102, "Agent": "TD3"}, {"env_step": 365000, "rew": 3068.6233, "rew_std": 691.8184, "Agent": "TD3"}, {"env_step": 370000, "rew": 3962.7737, "rew_std": 760.0341, "Agent": "TD3"}, {"env_step": 375000, "rew": 3594.7677, "rew_std": 928.1101, "Agent": "TD3"}, {"env_step": 380000, "rew": 3654.0992, "rew_std": 881.6398, "Agent": "TD3"}, {"env_step": 385000, "rew": 3912.8499, "rew_std": 945.1385, "Agent": "TD3"}, {"env_step": 390000, "rew": 3789.8911, "rew_std": 863.4987, "Agent": "TD3"}, {"env_step": 395000, "rew": 3682.2004, "rew_std": 1050.2498, "Agent": "TD3"}, {"env_step": 400000, "rew": 4042.171, "rew_std": 855.7865, "Agent": "TD3"}, {"env_step": 405000, "rew": 4160.8874, "rew_std": 634.3616, "Agent": "TD3"}, {"env_step": 410000, "rew": 3968.93, "rew_std": 772.6093, "Agent": "TD3"}, {"env_step": 415000, "rew": 4362.0422, "rew_std": 761.5106, "Agent": "TD3"}, {"env_step": 420000, "rew": 3947.9836, "rew_std": 735.9527, "Agent": "TD3"}, {"env_step": 425000, "rew": 4205.5062, "rew_std": 788.3009, "Agent": "TD3"}, {"env_step": 430000, "rew": 4308.0382, "rew_std": 395.7122, "Agent": "TD3"}, {"env_step": 435000, "rew": 4216.2622, "rew_std": 871.5691, "Agent": "TD3"}, {"env_step": 440000, "rew": 4050.0608, "rew_std": 577.2572, "Agent": "TD3"}, {"env_step": 445000, "rew": 4563.2002, "rew_std": 491.0323, "Agent": "TD3"}, {"env_step": 450000, "rew": 4183.4363, "rew_std": 674.3578, "Agent": "TD3"}, {"env_step": 455000, "rew": 4401.7438, "rew_std": 639.4418, "Agent": "TD3"}, {"env_step": 460000, "rew": 4497.9864, "rew_std": 751.2121, "Agent": "TD3"}, {"env_step": 465000, "rew": 4643.5722, "rew_std": 438.428, "Agent": "TD3"}, {"env_step": 470000, "rew": 4488.679, "rew_std": 531.0604, "Agent": "TD3"}, {"env_step": 475000, "rew": 4291.1472, "rew_std": 1132.5677, "Agent": "TD3"}, {"env_step": 480000, "rew": 4397.5194, "rew_std": 554.6465, "Agent": "TD3"}, {"env_step": 485000, "rew": 4631.6336, "rew_std": 421.5592, "Agent": "TD3"}, {"env_step": 490000, "rew": 4728.4588, "rew_std": 522.8134, "Agent": "TD3"}, {"env_step": 495000, "rew": 4670.8286, "rew_std": 437.9568, "Agent": "TD3"}, {"env_step": 500000, "rew": 4623.3424, "rew_std": 494.8101, "Agent": "TD3"}, {"env_step": 505000, "rew": 4941.599, "rew_std": 156.4723, "Agent": "TD3"}, {"env_step": 510000, "rew": 4498.4272, "rew_std": 492.6321, "Agent": "TD3"}, {"env_step": 515000, "rew": 4630.5692, "rew_std": 397.6329, "Agent": "TD3"}, {"env_step": 520000, "rew": 4826.7184, "rew_std": 320.7131, "Agent": "TD3"}, {"env_step": 525000, "rew": 4598.1875, "rew_std": 483.902, "Agent": "TD3"}, {"env_step": 530000, "rew": 4496.8694, "rew_std": 584.4539, "Agent": "TD3"}, {"env_step": 535000, "rew": 4623.5366, "rew_std": 351.8769, "Agent": "TD3"}, {"env_step": 540000, "rew": 4603.9342, "rew_std": 658.6824, "Agent": "TD3"}, {"env_step": 545000, "rew": 4682.8686, "rew_std": 562.6961, "Agent": "TD3"}, {"env_step": 550000, "rew": 4620.4423, "rew_std": 732.4667, "Agent": "TD3"}, {"env_step": 555000, "rew": 5009.661, "rew_std": 194.2122, "Agent": "TD3"}, {"env_step": 560000, "rew": 4688.508, "rew_std": 488.5095, "Agent": "TD3"}, {"env_step": 565000, "rew": 4785.6807, "rew_std": 295.0808, "Agent": "TD3"}, {"env_step": 570000, "rew": 4644.8156, "rew_std": 532.46, "Agent": "TD3"}, {"env_step": 575000, "rew": 4779.7482, "rew_std": 295.8876, "Agent": "TD3"}, {"env_step": 580000, "rew": 4770.5819, "rew_std": 491.2643, "Agent": "TD3"}, {"env_step": 585000, "rew": 4631.8668, "rew_std": 404.3156, "Agent": "TD3"}, {"env_step": 590000, "rew": 4815.5844, "rew_std": 477.5273, "Agent": "TD3"}, {"env_step": 595000, "rew": 4915.4074, "rew_std": 276.115, "Agent": "TD3"}, {"env_step": 600000, "rew": 4720.8844, "rew_std": 419.8293, "Agent": "TD3"}, {"env_step": 605000, "rew": 4830.0452, "rew_std": 425.9804, "Agent": "TD3"}, {"env_step": 610000, "rew": 4847.5045, "rew_std": 386.9902, "Agent": "TD3"}, {"env_step": 615000, "rew": 4924.1368, "rew_std": 266.5693, "Agent": "TD3"}, {"env_step": 620000, "rew": 4988.3792, "rew_std": 169.3005, "Agent": "TD3"}, {"env_step": 625000, "rew": 4724.847, "rew_std": 579.4092, "Agent": "TD3"}, {"env_step": 630000, "rew": 4761.8745, "rew_std": 440.2406, "Agent": "TD3"}, {"env_step": 635000, "rew": 4775.3046, "rew_std": 381.7462, "Agent": "TD3"}, {"env_step": 640000, "rew": 4946.9403, "rew_std": 383.9484, "Agent": "TD3"}, {"env_step": 645000, "rew": 4982.5448, "rew_std": 214.4079, "Agent": "TD3"}, {"env_step": 650000, "rew": 5004.9466, "rew_std": 150.9016, "Agent": "TD3"}, {"env_step": 655000, "rew": 4995.9812, "rew_std": 240.261, "Agent": "TD3"}, {"env_step": 660000, "rew": 4917.1866, "rew_std": 469.1764, "Agent": "TD3"}, {"env_step": 665000, "rew": 4801.5734, "rew_std": 506.8326, "Agent": "TD3"}, {"env_step": 670000, "rew": 4858.4089, "rew_std": 262.0223, "Agent": "TD3"}, {"env_step": 675000, "rew": 4995.3341, "rew_std": 190.7018, "Agent": "TD3"}, {"env_step": 680000, "rew": 4993.243, "rew_std": 256.0528, "Agent": "TD3"}, {"env_step": 685000, "rew": 4894.9504, "rew_std": 329.4103, "Agent": "TD3"}, {"env_step": 690000, "rew": 4836.4288, "rew_std": 333.3621, "Agent": "TD3"}, {"env_step": 695000, "rew": 4863.7781, "rew_std": 429.5135, "Agent": "TD3"}, {"env_step": 700000, "rew": 5047.3576, "rew_std": 176.8078, "Agent": "TD3"}, {"env_step": 705000, "rew": 4996.33, "rew_std": 201.191, "Agent": "TD3"}, {"env_step": 710000, "rew": 4880.6173, "rew_std": 230.9537, "Agent": "TD3"}, {"env_step": 715000, "rew": 5037.2655, "rew_std": 184.4567, "Agent": "TD3"}, {"env_step": 720000, "rew": 4857.1692, "rew_std": 454.8993, "Agent": "TD3"}, {"env_step": 725000, "rew": 4750.6899, "rew_std": 463.4295, "Agent": "TD3"}, {"env_step": 730000, "rew": 5040.6027, "rew_std": 215.3514, "Agent": "TD3"}, {"env_step": 735000, "rew": 4920.5525, "rew_std": 265.3201, "Agent": "TD3"}, {"env_step": 740000, "rew": 4908.3612, "rew_std": 355.8871, "Agent": "TD3"}, {"env_step": 745000, "rew": 5125.8994, "rew_std": 161.9509, "Agent": "TD3"}, {"env_step": 750000, "rew": 5000.1682, "rew_std": 244.1228, "Agent": "TD3"}, {"env_step": 755000, "rew": 5019.9781, "rew_std": 267.8087, "Agent": "TD3"}, {"env_step": 760000, "rew": 5121.22, "rew_std": 108.2708, "Agent": "TD3"}, {"env_step": 765000, "rew": 5064.4168, "rew_std": 237.7942, "Agent": "TD3"}, {"env_step": 770000, "rew": 5113.9521, "rew_std": 139.6293, "Agent": "TD3"}, {"env_step": 775000, "rew": 4808.9893, "rew_std": 458.4127, "Agent": "TD3"}, {"env_step": 780000, "rew": 5108.4583, "rew_std": 140.2175, "Agent": "TD3"}, {"env_step": 785000, "rew": 4874.2555, "rew_std": 306.674, "Agent": "TD3"}, {"env_step": 790000, "rew": 5114.0341, "rew_std": 222.1771, "Agent": "TD3"}, {"env_step": 795000, "rew": 4875.5048, "rew_std": 385.4428, "Agent": "TD3"}, {"env_step": 800000, "rew": 5079.8931, "rew_std": 246.6054, "Agent": "TD3"}, {"env_step": 805000, "rew": 5133.4923, "rew_std": 139.7789, "Agent": "TD3"}, {"env_step": 810000, "rew": 5050.1196, "rew_std": 288.2048, "Agent": "TD3"}, {"env_step": 815000, "rew": 5100.0691, "rew_std": 271.2653, "Agent": "TD3"}, {"env_step": 820000, "rew": 4971.7669, "rew_std": 269.1686, "Agent": "TD3"}, {"env_step": 825000, "rew": 4933.0076, "rew_std": 430.8825, "Agent": "TD3"}, {"env_step": 830000, "rew": 5101.0741, "rew_std": 113.8671, "Agent": "TD3"}, {"env_step": 835000, "rew": 5013.3863, "rew_std": 249.3497, "Agent": "TD3"}, {"env_step": 840000, "rew": 5001.234, "rew_std": 243.7169, "Agent": "TD3"}, {"env_step": 845000, "rew": 5018.7245, "rew_std": 199.6715, "Agent": "TD3"}, {"env_step": 850000, "rew": 4988.2064, "rew_std": 411.9275, "Agent": "TD3"}, {"env_step": 855000, "rew": 4938.9949, "rew_std": 352.3513, "Agent": "TD3"}, {"env_step": 860000, "rew": 5002.8016, "rew_std": 254.9775, "Agent": "TD3"}, {"env_step": 865000, "rew": 5017.9034, "rew_std": 171.6877, "Agent": "TD3"}, {"env_step": 870000, "rew": 4930.6844, "rew_std": 457.0508, "Agent": "TD3"}, {"env_step": 875000, "rew": 4979.0143, "rew_std": 219.4724, "Agent": "TD3"}, {"env_step": 880000, "rew": 4947.4845, "rew_std": 345.9994, "Agent": "TD3"}, {"env_step": 885000, "rew": 4908.4159, "rew_std": 484.5035, "Agent": "TD3"}, {"env_step": 890000, "rew": 4946.5147, "rew_std": 332.8934, "Agent": "TD3"}, {"env_step": 895000, "rew": 5160.1536, "rew_std": 207.4863, "Agent": "TD3"}, {"env_step": 900000, "rew": 5084.8653, "rew_std": 252.4938, "Agent": "TD3"}, {"env_step": 905000, "rew": 5131.8246, "rew_std": 158.9648, "Agent": "TD3"}, {"env_step": 910000, "rew": 5086.175, "rew_std": 151.9426, "Agent": "TD3"}, {"env_step": 915000, "rew": 5163.2683, "rew_std": 139.7717, "Agent": "TD3"}, {"env_step": 920000, "rew": 5113.859, "rew_std": 133.0161, "Agent": "TD3"}, {"env_step": 925000, "rew": 5114.2944, "rew_std": 214.1085, "Agent": "TD3"}, {"env_step": 930000, "rew": 5185.9206, "rew_std": 109.1179, "Agent": "TD3"}, {"env_step": 935000, "rew": 5128.1275, "rew_std": 169.5002, "Agent": "TD3"}, {"env_step": 940000, "rew": 4936.1542, "rew_std": 303.652, "Agent": "TD3"}, {"env_step": 945000, "rew": 5150.1266, "rew_std": 105.97, "Agent": "TD3"}, {"env_step": 950000, "rew": 5045.5078, "rew_std": 263.0376, "Agent": "TD3"}, {"env_step": 955000, "rew": 5106.9053, "rew_std": 143.1704, "Agent": "TD3"}, {"env_step": 960000, "rew": 4965.0764, "rew_std": 365.9184, "Agent": "TD3"}, {"env_step": 965000, "rew": 5094.2522, "rew_std": 240.831, "Agent": "TD3"}, {"env_step": 970000, "rew": 5142.6073, "rew_std": 128.5266, "Agent": "TD3"}, {"env_step": 975000, "rew": 5189.5491, "rew_std": 178.4785, "Agent": "TD3"}, {"env_step": 980000, "rew": 5005.5165, "rew_std": 513.0527, "Agent": "TD3"}, {"env_step": 985000, "rew": 5129.9301, "rew_std": 168.1549, "Agent": "TD3"}, {"env_step": 990000, "rew": 5030.1055, "rew_std": 318.9333, "Agent": "TD3"}, {"env_step": 995000, "rew": 5137.759, "rew_std": 203.0376, "Agent": "TD3"}, {"env_step": 1000000, "rew": 5043.2619, "rew_std": 286.2232, "Agent": "TD3"}, {"env_step": 1005000, "rew": 5153.1375, "rew_std": 176.2132, "Agent": "TD3"}, {"env_step": 1010000, "rew": 5121.4074, "rew_std": 239.9469, "Agent": "TD3"}, {"env_step": 1015000, "rew": 5029.4641, "rew_std": 281.9055, "Agent": "TD3"}, {"env_step": 1020000, "rew": 5119.6649, "rew_std": 238.2116, "Agent": "TD3"}, {"env_step": 1025000, "rew": 5093.208, "rew_std": 266.519, "Agent": "TD3"}, {"env_step": 1030000, "rew": 5063.8444, "rew_std": 267.4462, "Agent": "TD3"}, {"env_step": 1035000, "rew": 5060.6371, "rew_std": 270.9232, "Agent": "TD3"}, {"env_step": 1040000, "rew": 5145.6034, "rew_std": 225.2249, "Agent": "TD3"}, {"env_step": 1045000, "rew": 5199.8301, "rew_std": 118.791, "Agent": "TD3"}, {"env_step": 1050000, "rew": 5124.249, "rew_std": 271.596, "Agent": "TD3"}, {"env_step": 1055000, "rew": 5157.4669, "rew_std": 147.9857, "Agent": "TD3"}, {"env_step": 1060000, "rew": 5171.9615, "rew_std": 221.3771, "Agent": "TD3"}, {"env_step": 1065000, "rew": 5168.2077, "rew_std": 235.7423, "Agent": "TD3"}, {"env_step": 1070000, "rew": 5131.0357, "rew_std": 203.1461, "Agent": "TD3"}, {"env_step": 1075000, "rew": 5114.448, "rew_std": 254.8144, "Agent": "TD3"}, {"env_step": 1080000, "rew": 5188.6507, "rew_std": 171.5331, "Agent": "TD3"}, {"env_step": 1085000, "rew": 5094.6068, "rew_std": 242.4939, "Agent": "TD3"}, {"env_step": 1090000, "rew": 5178.7567, "rew_std": 117.6536, "Agent": "TD3"}, {"env_step": 1095000, "rew": 5134.6964, "rew_std": 241.2062, "Agent": "TD3"}, {"env_step": 1100000, "rew": 5021.4631, "rew_std": 283.4483, "Agent": "TD3"}, {"env_step": 1105000, "rew": 5129.941, "rew_std": 238.1083, "Agent": "TD3"}, {"env_step": 1110000, "rew": 5179.2979, "rew_std": 226.6365, "Agent": "TD3"}, {"env_step": 1115000, "rew": 5054.2752, "rew_std": 288.4716, "Agent": "TD3"}, {"env_step": 1120000, "rew": 5118.64, "rew_std": 223.76, "Agent": "TD3"}, {"env_step": 1125000, "rew": 5032.9734, "rew_std": 233.2601, "Agent": "TD3"}, {"env_step": 1130000, "rew": 5138.4733, "rew_std": 235.9326, "Agent": "TD3"}, {"env_step": 1135000, "rew": 4996.6592, "rew_std": 360.9811, "Agent": "TD3"}, {"env_step": 1140000, "rew": 5107.5643, "rew_std": 267.7652, "Agent": "TD3"}, {"env_step": 1145000, "rew": 4829.5349, "rew_std": 501.3194, "Agent": "TD3"}, {"env_step": 1150000, "rew": 5098.5737, "rew_std": 269.75, "Agent": "TD3"}, {"env_step": 1155000, "rew": 5140.4421, "rew_std": 219.9037, "Agent": "TD3"}, {"env_step": 1160000, "rew": 4887.2352, "rew_std": 754.7954, "Agent": "TD3"}, {"env_step": 1165000, "rew": 5194.6158, "rew_std": 238.8039, "Agent": "TD3"}, {"env_step": 1170000, "rew": 5103.5817, "rew_std": 252.1515, "Agent": "TD3"}, {"env_step": 1175000, "rew": 4956.8485, "rew_std": 407.4093, "Agent": "TD3"}, {"env_step": 1180000, "rew": 4921.0582, "rew_std": 443.066, "Agent": "TD3"}, {"env_step": 1185000, "rew": 5166.1484, "rew_std": 154.7587, "Agent": "TD3"}, {"env_step": 1190000, "rew": 5014.7964, "rew_std": 365.9749, "Agent": "TD3"}, {"env_step": 1195000, "rew": 4814.9366, "rew_std": 545.8645, "Agent": "TD3"}, {"env_step": 1200000, "rew": 5006.7689, "rew_std": 300.7244, "Agent": "TD3"}, {"env_step": 1205000, "rew": 5108.7124, "rew_std": 302.142, "Agent": "TD3"}, {"env_step": 1210000, "rew": 5088.0629, "rew_std": 416.3904, "Agent": "TD3"}, {"env_step": 1215000, "rew": 4947.1785, "rew_std": 551.1925, "Agent": "TD3"}, {"env_step": 1220000, "rew": 5089.226, "rew_std": 211.0536, "Agent": "TD3"}, {"env_step": 1225000, "rew": 4877.8198, "rew_std": 470.0478, "Agent": "TD3"}, {"env_step": 1230000, "rew": 5148.8341, "rew_std": 444.5354, "Agent": "TD3"}, {"env_step": 1235000, "rew": 5116.8597, "rew_std": 298.4278, "Agent": "TD3"}, {"env_step": 1240000, "rew": 5092.3513, "rew_std": 270.159, "Agent": "TD3"}, {"env_step": 1245000, "rew": 4793.7686, "rew_std": 1118.6323, "Agent": "TD3"}, {"env_step": 1250000, "rew": 4907.0573, "rew_std": 349.6552, "Agent": "TD3"}, {"env_step": 0, "rew": -46.1947, "rew_std": 25.5804, "Agent": "TRPO"}, {"env_step": 30720, "rew": -13.9012, "rew_std": 11.913, "Agent": "TRPO"}, {"env_step": 61440, "rew": 18.0615, "rew_std": 9.0166, "Agent": "TRPO"}, {"env_step": 92160, "rew": 51.9175, "rew_std": 16.4955, "Agent": "TRPO"}, {"env_step": 122880, "rew": 108.4958, "rew_std": 33.2241, "Agent": "TRPO"}, {"env_step": 153600, "rew": 183.9944, "rew_std": 35.8262, "Agent": "TRPO"}, {"env_step": 184320, "rew": 257.4861, "rew_std": 60.051, "Agent": "TRPO"}, {"env_step": 215040, "rew": 430.4068, "rew_std": 124.2541, "Agent": "TRPO"}, {"env_step": 245760, "rew": 564.2017, "rew_std": 146.0606, "Agent": "TRPO"}, {"env_step": 276480, "rew": 682.5168, "rew_std": 122.3187, "Agent": "TRPO"}, {"env_step": 307200, "rew": 832.5033, "rew_std": 159.2968, "Agent": "TRPO"}, {"env_step": 337920, "rew": 1014.1049, "rew_std": 221.8108, "Agent": "TRPO"}, {"env_step": 368640, "rew": 1128.5618, "rew_std": 167.9213, "Agent": "TRPO"}, {"env_step": 399360, "rew": 1146.4494, "rew_std": 215.1602, "Agent": "TRPO"}, {"env_step": 430080, "rew": 1233.6374, "rew_std": 286.8604, "Agent": "TRPO"}, {"env_step": 460800, "rew": 1261.4876, "rew_std": 301.0892, "Agent": "TRPO"}, {"env_step": 491520, "rew": 1472.6865, "rew_std": 215.7148, "Agent": "TRPO"}, {"env_step": 522240, "rew": 1501.5343, "rew_std": 428.1053, "Agent": "TRPO"}, {"env_step": 552960, "rew": 1631.5161, "rew_std": 558.1692, "Agent": "TRPO"}, {"env_step": 583680, "rew": 1741.6005, "rew_std": 483.7599, "Agent": "TRPO"}, {"env_step": 614400, "rew": 1809.3667, "rew_std": 444.8329, "Agent": "TRPO"}, {"env_step": 645120, "rew": 1943.2351, "rew_std": 386.0455, "Agent": "TRPO"}, {"env_step": 675840, "rew": 2119.3593, "rew_std": 463.9853, "Agent": "TRPO"}, {"env_step": 706560, "rew": 2160.9349, "rew_std": 299.8534, "Agent": "TRPO"}, {"env_step": 737280, "rew": 2205.6584, "rew_std": 495.8226, "Agent": "TRPO"}, {"env_step": 768000, "rew": 2148.4616, "rew_std": 617.4263, "Agent": "TRPO"}, {"env_step": 798720, "rew": 2294.2347, "rew_std": 619.654, "Agent": "TRPO"}, {"env_step": 829440, "rew": 2334.3877, "rew_std": 569.8366, "Agent": "TRPO"}, {"env_step": 860160, "rew": 2373.0195, "rew_std": 635.4872, "Agent": "TRPO"}, {"env_step": 890880, "rew": 2572.1874, "rew_std": 771.2781, "Agent": "TRPO"}, {"env_step": 921600, "rew": 2568.2581, "rew_std": 539.7726, "Agent": "TRPO"}, {"env_step": 952320, "rew": 2639.2867, "rew_std": 572.4301, "Agent": "TRPO"}, {"env_step": 983040, "rew": 2866.6508, "rew_std": 707.8937, "Agent": "TRPO"}, {"env_step": 1013760, "rew": 2731.0697, "rew_std": 675.644, "Agent": "TRPO"}, {"env_step": 1044480, "rew": 2702.1965, "rew_std": 742.4807, "Agent": "TRPO"}, {"env_step": 1075200, "rew": 3001.7621, "rew_std": 688.8314, "Agent": "TRPO"}, {"env_step": 1105920, "rew": 2877.8716, "rew_std": 922.2411, "Agent": "TRPO"}, {"env_step": 1136640, "rew": 3104.9072, "rew_std": 800.8227, "Agent": "TRPO"}, {"env_step": 1167360, "rew": 2976.5528, "rew_std": 661.5153, "Agent": "TRPO"}, {"env_step": 1198080, "rew": 3196.2228, "rew_std": 707.9679, "Agent": "TRPO"}, {"env_step": 1228800, "rew": 3237.2005, "rew_std": 505.6685, "Agent": "TRPO"}, {"env_step": 1259520, "rew": 3098.3083, "rew_std": 578.4346, "Agent": "TRPO"}, {"env_step": 1290240, "rew": 3356.962, "rew_std": 786.3838, "Agent": "TRPO"}, {"env_step": 1320960, "rew": 3241.2647, "rew_std": 600.3495, "Agent": "TRPO"}, {"env_step": 1351680, "rew": 3361.1722, "rew_std": 547.5151, "Agent": "TRPO"}, {"env_step": 1382400, "rew": 3495.3879, "rew_std": 662.5387, "Agent": "TRPO"}, {"env_step": 1413120, "rew": 3388.8443, "rew_std": 537.1016, "Agent": "TRPO"}, {"env_step": 1443840, "rew": 3493.6456, "rew_std": 509.2391, "Agent": "TRPO"}, {"env_step": 1474560, "rew": 3256.3625, "rew_std": 412.1903, "Agent": "TRPO"}, {"env_step": 1505280, "rew": 3695.3862, "rew_std": 644.5284, "Agent": "TRPO"}, {"env_step": 1536000, "rew": 3685.0253, "rew_std": 537.4812, "Agent": "TRPO"}, {"env_step": 1566720, "rew": 3565.3736, "rew_std": 647.2313, "Agent": "TRPO"}, {"env_step": 1597440, "rew": 3790.5153, "rew_std": 702.1943, "Agent": "TRPO"}, {"env_step": 1628160, "rew": 3598.3267, "rew_std": 626.2541, "Agent": "TRPO"}, {"env_step": 1658880, "rew": 3970.7287, "rew_std": 647.9836, "Agent": "TRPO"}, {"env_step": 1689600, "rew": 3658.6492, "rew_std": 645.0697, "Agent": "TRPO"}, {"env_step": 1720320, "rew": 3848.0345, "rew_std": 701.1273, "Agent": "TRPO"}, {"env_step": 1751040, "rew": 4061.4513, "rew_std": 690.0289, "Agent": "TRPO"}, {"env_step": 1781760, "rew": 3807.921, "rew_std": 706.5999, "Agent": "TRPO"}, {"env_step": 1812480, "rew": 3802.0342, "rew_std": 663.7139, "Agent": "TRPO"}, {"env_step": 1843200, "rew": 3873.6662, "rew_std": 566.3833, "Agent": "TRPO"}, {"env_step": 1873920, "rew": 4038.6082, "rew_std": 518.5079, "Agent": "TRPO"}, {"env_step": 1904640, "rew": 4167.3221, "rew_std": 440.9658, "Agent": "TRPO"}, {"env_step": 1935360, "rew": 3922.2855, "rew_std": 733.8238, "Agent": "TRPO"}, {"env_step": 1966080, "rew": 4174.7307, "rew_std": 495.1849, "Agent": "TRPO"}, {"env_step": 1996800, "rew": 4155.8384, "rew_std": 477.6376, "Agent": "TRPO"}, {"env_step": 2027520, "rew": 4259.3055, "rew_std": 577.3636, "Agent": "TRPO"}, {"env_step": 2058240, "rew": 4256.1933, "rew_std": 594.2826, "Agent": "TRPO"}, {"env_step": 2088960, "rew": 4355.0669, "rew_std": 605.8978, "Agent": "TRPO"}, {"env_step": 2119680, "rew": 4297.2252, "rew_std": 474.421, "Agent": "TRPO"}, {"env_step": 2150400, "rew": 4287.4698, "rew_std": 363.7332, "Agent": "TRPO"}, {"env_step": 2181120, "rew": 4398.2137, "rew_std": 354.3603, "Agent": "TRPO"}, {"env_step": 2211840, "rew": 4413.5796, "rew_std": 460.9681, "Agent": "TRPO"}, {"env_step": 2242560, "rew": 4329.001, "rew_std": 530.2755, "Agent": "TRPO"}, {"env_step": 2273280, "rew": 4252.3464, "rew_std": 597.5882, "Agent": "TRPO"}, {"env_step": 2304000, "rew": 4462.5657, "rew_std": 495.3802, "Agent": "TRPO"}, {"env_step": 2334720, "rew": 4374.2838, "rew_std": 457.0956, "Agent": "TRPO"}, {"env_step": 2365440, "rew": 4529.4549, "rew_std": 461.1733, "Agent": "TRPO"}, {"env_step": 2396160, "rew": 4291.4468, "rew_std": 411.2142, "Agent": "TRPO"}, {"env_step": 2426880, "rew": 4430.1459, "rew_std": 601.8812, "Agent": "TRPO"}, {"env_step": 2457600, "rew": 4508.078, "rew_std": 465.0507, "Agent": "TRPO"}, {"env_step": 2488320, "rew": 4151.3796, "rew_std": 414.656, "Agent": "TRPO"}, {"env_step": 2519040, "rew": 4705.7212, "rew_std": 477.1206, "Agent": "TRPO"}, {"env_step": 2549760, "rew": 4641.3111, "rew_std": 622.6369, "Agent": "TRPO"}, {"env_step": 2580480, "rew": 4484.1452, "rew_std": 442.0811, "Agent": "TRPO"}, {"env_step": 2611200, "rew": 4658.7294, "rew_std": 432.1496, "Agent": "TRPO"}, {"env_step": 2641920, "rew": 4625.8785, "rew_std": 357.4208, "Agent": "TRPO"}, {"env_step": 2672640, "rew": 4284.9794, "rew_std": 599.7262, "Agent": "TRPO"}, {"env_step": 2703360, "rew": 4419.1972, "rew_std": 447.5325, "Agent": "TRPO"}, {"env_step": 2734080, "rew": 4746.1067, "rew_std": 580.4515, "Agent": "TRPO"}, {"env_step": 2764800, "rew": 4756.6644, "rew_std": 502.4658, "Agent": "TRPO"}, {"env_step": 2795520, "rew": 4607.7287, "rew_std": 427.1984, "Agent": "TRPO"}, {"env_step": 2826240, "rew": 4404.4363, "rew_std": 555.3297, "Agent": "TRPO"}, {"env_step": 2856960, "rew": 4816.8623, "rew_std": 528.6794, "Agent": "TRPO"}, {"env_step": 2887680, "rew": 4753.3578, "rew_std": 715.7057, "Agent": "TRPO"}, {"env_step": 2918400, "rew": 4604.763, "rew_std": 447.4458, "Agent": "TRPO"}, {"env_step": 2949120, "rew": 4800.2693, "rew_std": 602.9667, "Agent": "TRPO"}, {"env_step": 2979840, "rew": 4772.9396, "rew_std": 368.1691, "Agent": "TRPO"}, {"env_step": 3010560, "rew": 4897.0257, "rew_std": 449.1329, "Agent": "TRPO"}, {"env_step": 3041280, "rew": 4841.7814, "rew_std": 534.8951, "Agent": "TRPO"}, {"env_step": 3072000, "rew": 4604.9043, "rew_std": 580.976, "Agent": "TRPO"}, {"env_step": 0, "rew": -46.1947, "rew_std": 25.5804, "Agent": "NPG"}, {"env_step": 30720, "rew": -29.0009, "rew_std": 15.8916, "Agent": "NPG"}, {"env_step": 61440, "rew": -10.4279, "rew_std": 11.0242, "Agent": "NPG"}, {"env_step": 92160, "rew": 11.1187, "rew_std": 9.0737, "Agent": "NPG"}, {"env_step": 122880, "rew": 27.5852, "rew_std": 16.4638, "Agent": "NPG"}, {"env_step": 153600, "rew": 56.7285, "rew_std": 23.8642, "Agent": "NPG"}, {"env_step": 184320, "rew": 81.2319, "rew_std": 29.1525, "Agent": "NPG"}, {"env_step": 215040, "rew": 159.4704, "rew_std": 47.3793, "Agent": "NPG"}, {"env_step": 245760, "rew": 174.9781, "rew_std": 57.693, "Agent": "NPG"}, {"env_step": 276480, "rew": 178.6358, "rew_std": 32.807, "Agent": "NPG"}, {"env_step": 307200, "rew": 218.3212, "rew_std": 41.035, "Agent": "NPG"}, {"env_step": 337920, "rew": 368.663, "rew_std": 80.0623, "Agent": "NPG"}, {"env_step": 368640, "rew": 432.2163, "rew_std": 63.1225, "Agent": "NPG"}, {"env_step": 399360, "rew": 568.0807, "rew_std": 125.517, "Agent": "NPG"}, {"env_step": 430080, "rew": 631.1259, "rew_std": 114.6371, "Agent": "NPG"}, {"env_step": 460800, "rew": 753.5929, "rew_std": 118.5185, "Agent": "NPG"}, {"env_step": 491520, "rew": 864.7371, "rew_std": 183.4071, "Agent": "NPG"}, {"env_step": 522240, "rew": 940.2978, "rew_std": 158.8018, "Agent": "NPG"}, {"env_step": 552960, "rew": 1021.7661, "rew_std": 159.2406, "Agent": "NPG"}, {"env_step": 583680, "rew": 1131.714, "rew_std": 150.4756, "Agent": "NPG"}, {"env_step": 614400, "rew": 1201.5181, "rew_std": 164.258, "Agent": "NPG"}, {"env_step": 645120, "rew": 1343.1191, "rew_std": 184.6686, "Agent": "NPG"}, {"env_step": 675840, "rew": 1318.401, "rew_std": 229.7468, "Agent": "NPG"}, {"env_step": 706560, "rew": 1459.1159, "rew_std": 222.6298, "Agent": "NPG"}, {"env_step": 737280, "rew": 1542.0295, "rew_std": 203.9479, "Agent": "NPG"}, {"env_step": 768000, "rew": 1634.9176, "rew_std": 218.2131, "Agent": "NPG"}, {"env_step": 798720, "rew": 1692.0983, "rew_std": 343.9785, "Agent": "NPG"}, {"env_step": 829440, "rew": 1906.1507, "rew_std": 457.5102, "Agent": "NPG"}, {"env_step": 860160, "rew": 1775.1748, "rew_std": 342.0144, "Agent": "NPG"}, {"env_step": 890880, "rew": 2102.0494, "rew_std": 355.8503, "Agent": "NPG"}, {"env_step": 921600, "rew": 2046.7794, "rew_std": 340.8884, "Agent": "NPG"}, {"env_step": 952320, "rew": 2164.3512, "rew_std": 405.7482, "Agent": "NPG"}, {"env_step": 983040, "rew": 2348.312, "rew_std": 509.9401, "Agent": "NPG"}, {"env_step": 1013760, "rew": 2365.8958, "rew_std": 523.6603, "Agent": "NPG"}, {"env_step": 1044480, "rew": 2324.9164, "rew_std": 528.4526, "Agent": "NPG"}, {"env_step": 1075200, "rew": 2493.722, "rew_std": 583.8895, "Agent": "NPG"}, {"env_step": 1105920, "rew": 2638.7282, "rew_std": 650.3356, "Agent": "NPG"}, {"env_step": 1136640, "rew": 2588.3598, "rew_std": 506.4239, "Agent": "NPG"}, {"env_step": 1167360, "rew": 2657.8218, "rew_std": 542.1361, "Agent": "NPG"}, {"env_step": 1198080, "rew": 2772.0887, "rew_std": 399.8722, "Agent": "NPG"}, {"env_step": 1228800, "rew": 2983.8532, "rew_std": 405.1427, "Agent": "NPG"}, {"env_step": 1259520, "rew": 2969.912, "rew_std": 543.0224, "Agent": "NPG"}, {"env_step": 1290240, "rew": 2898.5856, "rew_std": 687.1927, "Agent": "NPG"}, {"env_step": 1320960, "rew": 2979.687, "rew_std": 685.3181, "Agent": "NPG"}, {"env_step": 1351680, "rew": 2937.7865, "rew_std": 654.2811, "Agent": "NPG"}, {"env_step": 1382400, "rew": 3180.3262, "rew_std": 502.1675, "Agent": "NPG"}, {"env_step": 1413120, "rew": 3181.4746, "rew_std": 526.9918, "Agent": "NPG"}, {"env_step": 1443840, "rew": 3186.725, "rew_std": 483.0104, "Agent": "NPG"}, {"env_step": 1474560, "rew": 3347.1624, "rew_std": 474.3212, "Agent": "NPG"}, {"env_step": 1505280, "rew": 3264.1247, "rew_std": 542.1326, "Agent": "NPG"}, {"env_step": 1536000, "rew": 3267.3737, "rew_std": 536.477, "Agent": "NPG"}, {"env_step": 1566720, "rew": 3677.709, "rew_std": 486.0857, "Agent": "NPG"}, {"env_step": 1597440, "rew": 3379.1083, "rew_std": 683.1763, "Agent": "NPG"}, {"env_step": 1628160, "rew": 3533.6653, "rew_std": 437.9623, "Agent": "NPG"}, {"env_step": 1658880, "rew": 3534.0292, "rew_std": 787.6867, "Agent": "NPG"}, {"env_step": 1689600, "rew": 3399.7589, "rew_std": 685.1419, "Agent": "NPG"}, {"env_step": 1720320, "rew": 3587.7024, "rew_std": 471.134, "Agent": "NPG"}, {"env_step": 1751040, "rew": 3560.6233, "rew_std": 616.8093, "Agent": "NPG"}, {"env_step": 1781760, "rew": 3635.7303, "rew_std": 608.4666, "Agent": "NPG"}, {"env_step": 1812480, "rew": 3713.5899, "rew_std": 802.8577, "Agent": "NPG"}, {"env_step": 1843200, "rew": 3590.908, "rew_std": 557.7779, "Agent": "NPG"}, {"env_step": 1873920, "rew": 3903.7812, "rew_std": 655.7805, "Agent": "NPG"}, {"env_step": 1904640, "rew": 3770.5075, "rew_std": 645.9229, "Agent": "NPG"}, {"env_step": 1935360, "rew": 4110.3599, "rew_std": 657.3085, "Agent": "NPG"}, {"env_step": 1966080, "rew": 3851.8914, "rew_std": 745.2079, "Agent": "NPG"}, {"env_step": 1996800, "rew": 3998.0014, "rew_std": 650.7886, "Agent": "NPG"}, {"env_step": 2027520, "rew": 3739.3657, "rew_std": 715.6727, "Agent": "NPG"}, {"env_step": 2058240, "rew": 4230.5764, "rew_std": 578.5901, "Agent": "NPG"}, {"env_step": 2088960, "rew": 4140.935, "rew_std": 624.3245, "Agent": "NPG"}, {"env_step": 2119680, "rew": 4228.9808, "rew_std": 573.9637, "Agent": "NPG"}, {"env_step": 2150400, "rew": 4142.6649, "rew_std": 419.5781, "Agent": "NPG"}, {"env_step": 2181120, "rew": 4089.8147, "rew_std": 302.9776, "Agent": "NPG"}, {"env_step": 2211840, "rew": 4276.8854, "rew_std": 674.4424, "Agent": "NPG"}, {"env_step": 2242560, "rew": 4312.1654, "rew_std": 567.5481, "Agent": "NPG"}, {"env_step": 2273280, "rew": 4224.3566, "rew_std": 445.0954, "Agent": "NPG"}, {"env_step": 2304000, "rew": 3878.7632, "rew_std": 866.3551, "Agent": "NPG"}, {"env_step": 2334720, "rew": 4001.3801, "rew_std": 478.1705, "Agent": "NPG"}, {"env_step": 2365440, "rew": 4284.5594, "rew_std": 448.8121, "Agent": "NPG"}, {"env_step": 2396160, "rew": 4406.6923, "rew_std": 516.8683, "Agent": "NPG"}, {"env_step": 2426880, "rew": 4145.9032, "rew_std": 519.4268, "Agent": "NPG"}, {"env_step": 2457600, "rew": 4241.7632, "rew_std": 685.6791, "Agent": "NPG"}, {"env_step": 2488320, "rew": 4154.1219, "rew_std": 437.1567, "Agent": "NPG"}, {"env_step": 2519040, "rew": 4495.7374, "rew_std": 449.1806, "Agent": "NPG"}, {"env_step": 2549760, "rew": 4289.7487, "rew_std": 526.7084, "Agent": "NPG"}, {"env_step": 2580480, "rew": 4246.0211, "rew_std": 530.6265, "Agent": "NPG"}, {"env_step": 2611200, "rew": 4463.8985, "rew_std": 572.8797, "Agent": "NPG"}, {"env_step": 2641920, "rew": 4298.6282, "rew_std": 665.9416, "Agent": "NPG"}, {"env_step": 2672640, "rew": 4485.5394, "rew_std": 359.9158, "Agent": "NPG"}, {"env_step": 2703360, "rew": 4820.597, "rew_std": 318.5627, "Agent": "NPG"}, {"env_step": 2734080, "rew": 4540.1727, "rew_std": 412.9545, "Agent": "NPG"}, {"env_step": 2764800, "rew": 4500.1939, "rew_std": 596.1343, "Agent": "NPG"}, {"env_step": 2795520, "rew": 4664.0692, "rew_std": 426.6286, "Agent": "NPG"}, {"env_step": 2826240, "rew": 4516.467, "rew_std": 435.656, "Agent": "NPG"}, {"env_step": 2856960, "rew": 4545.139, "rew_std": 496.411, "Agent": "NPG"}, {"env_step": 2887680, "rew": 4444.0518, "rew_std": 596.6213, "Agent": "NPG"}, {"env_step": 2918400, "rew": 4658.8307, "rew_std": 652.0561, "Agent": "NPG"}, {"env_step": 2949120, "rew": 4509.4439, "rew_std": 466.3483, "Agent": "NPG"}, {"env_step": 2979840, "rew": 4589.8375, "rew_std": 414.0363, "Agent": "NPG"}, {"env_step": 3010560, "rew": 4693.0902, "rew_std": 490.4674, "Agent": "NPG"}, {"env_step": 3041280, "rew": 4678.8152, "rew_std": 313.0054, "Agent": "NPG"}, {"env_step": 3072000, "rew": 4736.6273, "rew_std": 376.7205, "Agent": "NPG"}, {"env_step": 0, "rew": 130.0823, "rew_std": 15.1427, "Agent": "PPO"}, {"env_step": 30720, "rew": 350.9917, "rew_std": 24.4802, "Agent": "PPO"}, {"env_step": 61440, "rew": 425.4232, "rew_std": 23.1423, "Agent": "PPO"}, {"env_step": 92160, "rew": 465.9451, "rew_std": 49.5543, "Agent": "PPO"}, {"env_step": 122880, "rew": 493.9147, "rew_std": 26.3191, "Agent": "PPO"}, {"env_step": 153600, "rew": 490.4975, "rew_std": 42.5076, "Agent": "PPO"}, {"env_step": 184320, "rew": 501.0533, "rew_std": 44.1211, "Agent": "PPO"}, {"env_step": 215040, "rew": 511.8286, "rew_std": 45.6841, "Agent": "PPO"}, {"env_step": 245760, "rew": 510.9731, "rew_std": 39.5026, "Agent": "PPO"}, {"env_step": 276480, "rew": 521.749, "rew_std": 45.4886, "Agent": "PPO"}, {"env_step": 307200, "rew": 541.7495, "rew_std": 33.5348, "Agent": "PPO"}, {"env_step": 337920, "rew": 551.0738, "rew_std": 41.6129, "Agent": "PPO"}, {"env_step": 368640, "rew": 558.4841, "rew_std": 42.2806, "Agent": "PPO"}, {"env_step": 399360, "rew": 559.4836, "rew_std": 61.2039, "Agent": "PPO"}, {"env_step": 430080, "rew": 567.995, "rew_std": 38.5396, "Agent": "PPO"}, {"env_step": 460800, "rew": 550.0916, "rew_std": 39.8131, "Agent": "PPO"}, {"env_step": 491520, "rew": 582.7719, "rew_std": 64.735, "Agent": "PPO"}, {"env_step": 522240, "rew": 579.7875, "rew_std": 48.9013, "Agent": "PPO"}, {"env_step": 552960, "rew": 606.7364, "rew_std": 78.9236, "Agent": "PPO"}, {"env_step": 583680, "rew": 609.9051, "rew_std": 50.6975, "Agent": "PPO"}, {"env_step": 614400, "rew": 631.2371, "rew_std": 77.1233, "Agent": "PPO"}, {"env_step": 645120, "rew": 648.6753, "rew_std": 92.364, "Agent": "PPO"}, {"env_step": 675840, "rew": 634.2489, "rew_std": 73.4048, "Agent": "PPO"}, {"env_step": 706560, "rew": 628.1442, "rew_std": 72.5634, "Agent": "PPO"}, {"env_step": 737280, "rew": 663.1003, "rew_std": 80.5057, "Agent": "PPO"}, {"env_step": 768000, "rew": 676.9848, "rew_std": 73.3874, "Agent": "PPO"}, {"env_step": 798720, "rew": 656.3851, "rew_std": 78.3265, "Agent": "PPO"}, {"env_step": 829440, "rew": 712.8563, "rew_std": 67.9891, "Agent": "PPO"}, {"env_step": 860160, "rew": 701.9783, "rew_std": 119.4269, "Agent": "PPO"}, {"env_step": 890880, "rew": 727.9524, "rew_std": 98.1045, "Agent": "PPO"}, {"env_step": 921600, "rew": 716.8859, "rew_std": 90.7377, "Agent": "PPO"}, {"env_step": 952320, "rew": 739.1209, "rew_std": 132.9358, "Agent": "PPO"}, {"env_step": 983040, "rew": 787.0681, "rew_std": 193.5017, "Agent": "PPO"}, {"env_step": 1013760, "rew": 732.3068, "rew_std": 105.6359, "Agent": "PPO"}, {"env_step": 1044480, "rew": 747.3812, "rew_std": 144.6905, "Agent": "PPO"}, {"env_step": 1075200, "rew": 753.0065, "rew_std": 150.7269, "Agent": "PPO"}, {"env_step": 1105920, "rew": 767.1866, "rew_std": 177.7926, "Agent": "PPO"}, {"env_step": 1136640, "rew": 745.1658, "rew_std": 115.0446, "Agent": "PPO"}, {"env_step": 1167360, "rew": 775.4483, "rew_std": 129.888, "Agent": "PPO"}, {"env_step": 1198080, "rew": 784.5837, "rew_std": 148.4, "Agent": "PPO"}, {"env_step": 1228800, "rew": 825.0454, "rew_std": 157.0856, "Agent": "PPO"}, {"env_step": 1259520, "rew": 829.8744, "rew_std": 171.9486, "Agent": "PPO"}, {"env_step": 1290240, "rew": 826.0339, "rew_std": 164.3613, "Agent": "PPO"}, {"env_step": 1320960, "rew": 804.6047, "rew_std": 151.2456, "Agent": "PPO"}, {"env_step": 1351680, "rew": 815.0404, "rew_std": 188.6186, "Agent": "PPO"}, {"env_step": 1382400, "rew": 835.5702, "rew_std": 150.8863, "Agent": "PPO"}, {"env_step": 1413120, "rew": 842.724, "rew_std": 188.133, "Agent": "PPO"}, {"env_step": 1443840, "rew": 860.3203, "rew_std": 198.2533, "Agent": "PPO"}, {"env_step": 1474560, "rew": 841.7917, "rew_std": 155.3849, "Agent": "PPO"}, {"env_step": 1505280, "rew": 800.4889, "rew_std": 148.3306, "Agent": "PPO"}, {"env_step": 1536000, "rew": 935.7042, "rew_std": 249.8531, "Agent": "PPO"}, {"env_step": 1566720, "rew": 888.7137, "rew_std": 174.3786, "Agent": "PPO"}, {"env_step": 1597440, "rew": 832.1629, "rew_std": 155.7721, "Agent": "PPO"}, {"env_step": 1628160, "rew": 986.3306, "rew_std": 285.295, "Agent": "PPO"}, {"env_step": 1658880, "rew": 853.313, "rew_std": 158.3305, "Agent": "PPO"}, {"env_step": 1689600, "rew": 877.4147, "rew_std": 171.3564, "Agent": "PPO"}, {"env_step": 1720320, "rew": 925.5333, "rew_std": 274.6513, "Agent": "PPO"}, {"env_step": 1751040, "rew": 920.2305, "rew_std": 239.9945, "Agent": "PPO"}, {"env_step": 1781760, "rew": 927.6438, "rew_std": 268.7642, "Agent": "PPO"}, {"env_step": 1812480, "rew": 965.5925, "rew_std": 252.3835, "Agent": "PPO"}, {"env_step": 1843200, "rew": 977.3575, "rew_std": 338.2636, "Agent": "PPO"}, {"env_step": 1873920, "rew": 1062.624, "rew_std": 421.2998, "Agent": "PPO"}, {"env_step": 1904640, "rew": 1017.8632, "rew_std": 357.3262, "Agent": "PPO"}, {"env_step": 1935360, "rew": 1045.3381, "rew_std": 363.1296, "Agent": "PPO"}, {"env_step": 1966080, "rew": 988.6769, "rew_std": 293.54, "Agent": "PPO"}, {"env_step": 1996800, "rew": 1039.9539, "rew_std": 317.9886, "Agent": "PPO"}, {"env_step": 2027520, "rew": 1000.3818, "rew_std": 267.0232, "Agent": "PPO"}, {"env_step": 2058240, "rew": 932.8863, "rew_std": 172.0988, "Agent": "PPO"}, {"env_step": 2088960, "rew": 1063.8193, "rew_std": 356.9004, "Agent": "PPO"}, {"env_step": 2119680, "rew": 1007.1344, "rew_std": 293.2305, "Agent": "PPO"}, {"env_step": 2150400, "rew": 1118.3303, "rew_std": 434.818, "Agent": "PPO"}, {"env_step": 2181120, "rew": 1040.4434, "rew_std": 270.6467, "Agent": "PPO"}, {"env_step": 2211840, "rew": 1104.2401, "rew_std": 395.4114, "Agent": "PPO"}, {"env_step": 2242560, "rew": 1083.2706, "rew_std": 306.5721, "Agent": "PPO"}, {"env_step": 2273280, "rew": 1027.5872, "rew_std": 275.1138, "Agent": "PPO"}, {"env_step": 2304000, "rew": 1177.9164, "rew_std": 353.1948, "Agent": "PPO"}, {"env_step": 2334720, "rew": 1087.1684, "rew_std": 305.1818, "Agent": "PPO"}, {"env_step": 2365440, "rew": 1177.7415, "rew_std": 304.9442, "Agent": "PPO"}, {"env_step": 2396160, "rew": 1058.5766, "rew_std": 268.4139, "Agent": "PPO"}, {"env_step": 2426880, "rew": 1219.0824, "rew_std": 504.7934, "Agent": "PPO"}, {"env_step": 2457600, "rew": 1138.8759, "rew_std": 339.5198, "Agent": "PPO"}, {"env_step": 2488320, "rew": 1154.2304, "rew_std": 377.9643, "Agent": "PPO"}, {"env_step": 2519040, "rew": 1290.3548, "rew_std": 410.5781, "Agent": "PPO"}, {"env_step": 2549760, "rew": 1251.92, "rew_std": 435.415, "Agent": "PPO"}, {"env_step": 2580480, "rew": 1235.0206, "rew_std": 365.7419, "Agent": "PPO"}, {"env_step": 2611200, "rew": 1273.9936, "rew_std": 443.1518, "Agent": "PPO"}, {"env_step": 2641920, "rew": 1222.3642, "rew_std": 409.0311, "Agent": "PPO"}, {"env_step": 2672640, "rew": 1348.982, "rew_std": 560.3245, "Agent": "PPO"}, {"env_step": 2703360, "rew": 1274.8993, "rew_std": 312.1054, "Agent": "PPO"}, {"env_step": 2734080, "rew": 1359.7101, "rew_std": 572.6597, "Agent": "PPO"}, {"env_step": 2764800, "rew": 1323.7258, "rew_std": 509.658, "Agent": "PPO"}, {"env_step": 2795520, "rew": 1181.5144, "rew_std": 341.507, "Agent": "PPO"}, {"env_step": 2826240, "rew": 1279.5406, "rew_std": 448.5542, "Agent": "PPO"}, {"env_step": 2856960, "rew": 1291.4052, "rew_std": 369.7179, "Agent": "PPO"}, {"env_step": 2887680, "rew": 1144.7832, "rew_std": 223.4122, "Agent": "PPO"}, {"env_step": 2918400, "rew": 1323.9516, "rew_std": 476.6721, "Agent": "PPO"}, {"env_step": 2949120, "rew": 1208.2497, "rew_std": 322.735, "Agent": "PPO"}, {"env_step": 2979840, "rew": 1346.5563, "rew_std": 454.0205, "Agent": "PPO"}, {"env_step": 3010560, "rew": 1418.4356, "rew_std": 586.806, "Agent": "PPO"}, {"env_step": 3041280, "rew": 1401.6184, "rew_std": 675.0394, "Agent": "PPO"}, {"env_step": 3072000, "rew": 1225.8906, "rew_std": 338.3397, "Agent": "PPO"}, {"env_step": 0, "rew": 134.2851, "rew_std": 8.9071, "Agent": "REINFORCE"}, {"env_step": 30720, "rew": 132.3317, "rew_std": 11.5884, "Agent": "REINFORCE"}, {"env_step": 61440, "rew": 145.5181, "rew_std": 8.2425, "Agent": "REINFORCE"}, {"env_step": 92160, "rew": 160.6736, "rew_std": 20.2516, "Agent": "REINFORCE"}, {"env_step": 122880, "rew": 182.995, "rew_std": 21.3789, "Agent": "REINFORCE"}, {"env_step": 153600, "rew": 229.754, "rew_std": 36.6985, "Agent": "REINFORCE"}, {"env_step": 184320, "rew": 279.1829, "rew_std": 34.0526, "Agent": "REINFORCE"}, {"env_step": 215040, "rew": 298.9748, "rew_std": 51.2942, "Agent": "REINFORCE"}, {"env_step": 245760, "rew": 347.8587, "rew_std": 34.0078, "Agent": "REINFORCE"}, {"env_step": 276480, "rew": 346.622, "rew_std": 40.2387, "Agent": "REINFORCE"}, {"env_step": 307200, "rew": 356.9964, "rew_std": 31.2667, "Agent": "REINFORCE"}, {"env_step": 337920, "rew": 348.9647, "rew_std": 42.06, "Agent": "REINFORCE"}, {"env_step": 368640, "rew": 357.4741, "rew_std": 44.106, "Agent": "REINFORCE"}, {"env_step": 399360, "rew": 369.9218, "rew_std": 54.5148, "Agent": "REINFORCE"}, {"env_step": 430080, "rew": 390.0737, "rew_std": 44.8478, "Agent": "REINFORCE"}, {"env_step": 460800, "rew": 381.269, "rew_std": 48.6122, "Agent": "REINFORCE"}, {"env_step": 491520, "rew": 362.2927, "rew_std": 40.5626, "Agent": "REINFORCE"}, {"env_step": 522240, "rew": 376.2153, "rew_std": 48.4882, "Agent": "REINFORCE"}, {"env_step": 552960, "rew": 392.1445, "rew_std": 49.4761, "Agent": "REINFORCE"}, {"env_step": 583680, "rew": 385.8984, "rew_std": 51.5347, "Agent": "REINFORCE"}, {"env_step": 614400, "rew": 379.0199, "rew_std": 37.4646, "Agent": "REINFORCE"}, {"env_step": 645120, "rew": 384.7891, "rew_std": 28.1911, "Agent": "REINFORCE"}, {"env_step": 675840, "rew": 370.1497, "rew_std": 46.0869, "Agent": "REINFORCE"}, {"env_step": 706560, "rew": 382.5503, "rew_std": 30.2578, "Agent": "REINFORCE"}, {"env_step": 737280, "rew": 392.037, "rew_std": 35.1496, "Agent": "REINFORCE"}, {"env_step": 768000, "rew": 385.1342, "rew_std": 54.9051, "Agent": "REINFORCE"}, {"env_step": 798720, "rew": 373.2024, "rew_std": 46.5376, "Agent": "REINFORCE"}, {"env_step": 829440, "rew": 382.3954, "rew_std": 54.6306, "Agent": "REINFORCE"}, {"env_step": 860160, "rew": 370.3266, "rew_std": 56.9917, "Agent": "REINFORCE"}, {"env_step": 890880, "rew": 374.9132, "rew_std": 60.8034, "Agent": "REINFORCE"}, {"env_step": 921600, "rew": 372.2838, "rew_std": 55.2198, "Agent": "REINFORCE"}, {"env_step": 952320, "rew": 363.9386, "rew_std": 44.3701, "Agent": "REINFORCE"}, {"env_step": 983040, "rew": 393.1796, "rew_std": 53.9256, "Agent": "REINFORCE"}, {"env_step": 1013760, "rew": 391.3388, "rew_std": 41.1072, "Agent": "REINFORCE"}, {"env_step": 1044480, "rew": 378.9883, "rew_std": 50.5282, "Agent": "REINFORCE"}, {"env_step": 1075200, "rew": 377.087, "rew_std": 56.9021, "Agent": "REINFORCE"}, {"env_step": 1105920, "rew": 363.5972, "rew_std": 51.4931, "Agent": "REINFORCE"}, {"env_step": 1136640, "rew": 399.5731, "rew_std": 55.8625, "Agent": "REINFORCE"}, {"env_step": 1167360, "rew": 405.3753, "rew_std": 62.2749, "Agent": "REINFORCE"}, {"env_step": 1198080, "rew": 402.9043, "rew_std": 53.5287, "Agent": "REINFORCE"}, {"env_step": 1228800, "rew": 414.8761, "rew_std": 38.5687, "Agent": "REINFORCE"}, {"env_step": 1259520, "rew": 399.3817, "rew_std": 50.0419, "Agent": "REINFORCE"}, {"env_step": 1290240, "rew": 398.1147, "rew_std": 57.708, "Agent": "REINFORCE"}, {"env_step": 1320960, "rew": 380.6883, "rew_std": 54.8462, "Agent": "REINFORCE"}, {"env_step": 1351680, "rew": 396.672, "rew_std": 56.6795, "Agent": "REINFORCE"}, {"env_step": 1382400, "rew": 386.1521, "rew_std": 58.6364, "Agent": "REINFORCE"}, {"env_step": 1413120, "rew": 367.3148, "rew_std": 41.4321, "Agent": "REINFORCE"}, {"env_step": 1443840, "rew": 372.0875, "rew_std": 50.2092, "Agent": "REINFORCE"}, {"env_step": 1474560, "rew": 382.4313, "rew_std": 48.7902, "Agent": "REINFORCE"}, {"env_step": 1505280, "rew": 396.4733, "rew_std": 44.2938, "Agent": "REINFORCE"}, {"env_step": 1536000, "rew": 380.5775, "rew_std": 46.8464, "Agent": "REINFORCE"}, {"env_step": 1566720, "rew": 382.3344, "rew_std": 28.7958, "Agent": "REINFORCE"}, {"env_step": 1597440, "rew": 382.907, "rew_std": 40.5035, "Agent": "REINFORCE"}, {"env_step": 1628160, "rew": 406.9931, "rew_std": 37.8681, "Agent": "REINFORCE"}, {"env_step": 1658880, "rew": 413.1177, "rew_std": 62.0168, "Agent": "REINFORCE"}, {"env_step": 1689600, "rew": 401.5583, "rew_std": 20.9659, "Agent": "REINFORCE"}, {"env_step": 1720320, "rew": 391.8222, "rew_std": 27.7617, "Agent": "REINFORCE"}, {"env_step": 1751040, "rew": 423.1126, "rew_std": 38.2972, "Agent": "REINFORCE"}, {"env_step": 1781760, "rew": 412.7687, "rew_std": 37.9509, "Agent": "REINFORCE"}, {"env_step": 1812480, "rew": 397.8555, "rew_std": 32.0286, "Agent": "REINFORCE"}, {"env_step": 1843200, "rew": 399.915, "rew_std": 26.438, "Agent": "REINFORCE"}, {"env_step": 1873920, "rew": 438.2131, "rew_std": 47.8379, "Agent": "REINFORCE"}, {"env_step": 1904640, "rew": 388.0629, "rew_std": 37.9841, "Agent": "REINFORCE"}, {"env_step": 1935360, "rew": 405.3549, "rew_std": 47.5364, "Agent": "REINFORCE"}, {"env_step": 1966080, "rew": 400.3994, "rew_std": 48.1861, "Agent": "REINFORCE"}, {"env_step": 1996800, "rew": 421.4098, "rew_std": 58.5822, "Agent": "REINFORCE"}, {"env_step": 2027520, "rew": 419.1956, "rew_std": 60.8501, "Agent": "REINFORCE"}, {"env_step": 2058240, "rew": 404.4959, "rew_std": 39.7589, "Agent": "REINFORCE"}, {"env_step": 2088960, "rew": 411.0288, "rew_std": 45.8849, "Agent": "REINFORCE"}, {"env_step": 2119680, "rew": 403.3878, "rew_std": 45.458, "Agent": "REINFORCE"}, {"env_step": 2150400, "rew": 402.6161, "rew_std": 44.98, "Agent": "REINFORCE"}, {"env_step": 2181120, "rew": 402.5555, "rew_std": 37.1715, "Agent": "REINFORCE"}, {"env_step": 2211840, "rew": 405.1573, "rew_std": 46.4707, "Agent": "REINFORCE"}, {"env_step": 2242560, "rew": 405.7697, "rew_std": 51.8826, "Agent": "REINFORCE"}, {"env_step": 2273280, "rew": 399.8488, "rew_std": 45.9409, "Agent": "REINFORCE"}, {"env_step": 2304000, "rew": 400.3283, "rew_std": 37.1483, "Agent": "REINFORCE"}, {"env_step": 2334720, "rew": 404.6552, "rew_std": 43.07, "Agent": "REINFORCE"}, {"env_step": 2365440, "rew": 417.5301, "rew_std": 57.2134, "Agent": "REINFORCE"}, {"env_step": 2396160, "rew": 422.0997, "rew_std": 46.2711, "Agent": "REINFORCE"}, {"env_step": 2426880, "rew": 421.0184, "rew_std": 50.7304, "Agent": "REINFORCE"}, {"env_step": 2457600, "rew": 417.0316, "rew_std": 41.817, "Agent": "REINFORCE"}, {"env_step": 2488320, "rew": 398.6818, "rew_std": 42.1057, "Agent": "REINFORCE"}, {"env_step": 2519040, "rew": 403.3902, "rew_std": 34.9101, "Agent": "REINFORCE"}, {"env_step": 2549760, "rew": 413.0594, "rew_std": 57.6956, "Agent": "REINFORCE"}, {"env_step": 2580480, "rew": 391.2237, "rew_std": 43.119, "Agent": "REINFORCE"}, {"env_step": 2611200, "rew": 398.634, "rew_std": 38.1579, "Agent": "REINFORCE"}, {"env_step": 2641920, "rew": 414.8102, "rew_std": 55.8519, "Agent": "REINFORCE"}, {"env_step": 2672640, "rew": 428.2386, "rew_std": 61.3671, "Agent": "REINFORCE"}, {"env_step": 2703360, "rew": 413.7839, "rew_std": 53.886, "Agent": "REINFORCE"}, {"env_step": 2734080, "rew": 409.3724, "rew_std": 58.9423, "Agent": "REINFORCE"}, {"env_step": 2764800, "rew": 418.4324, "rew_std": 41.7632, "Agent": "REINFORCE"}, {"env_step": 2795520, "rew": 411.0972, "rew_std": 53.0598, "Agent": "REINFORCE"}, {"env_step": 2826240, "rew": 385.6242, "rew_std": 45.5054, "Agent": "REINFORCE"}, {"env_step": 2856960, "rew": 409.6031, "rew_std": 46.3376, "Agent": "REINFORCE"}, {"env_step": 2887680, "rew": 401.2584, "rew_std": 62.2796, "Agent": "REINFORCE"}, {"env_step": 2918400, "rew": 401.0642, "rew_std": 58.2344, "Agent": "REINFORCE"}, {"env_step": 2949120, "rew": 400.8198, "rew_std": 67.4508, "Agent": "REINFORCE"}, {"env_step": 2979840, "rew": 399.4039, "rew_std": 53.3394, "Agent": "REINFORCE"}, {"env_step": 3010560, "rew": 389.8431, "rew_std": 39.3272, "Agent": "REINFORCE"}, {"env_step": 3041280, "rew": 394.067, "rew_std": 47.9829, "Agent": "REINFORCE"}, {"env_step": 3072000, "rew": 387.9552, "rew_std": 49.9772, "Agent": "REINFORCE"}, {"env_step": 3102720, "rew": 387.1675, "rew_std": 65.9888, "Agent": "REINFORCE"}, {"env_step": 3133440, "rew": 378.6828, "rew_std": 64.4292, "Agent": "REINFORCE"}, {"env_step": 3164160, "rew": 393.6185, "rew_std": 45.4058, "Agent": "REINFORCE"}, {"env_step": 3194880, "rew": 394.6443, "rew_std": 46.5393, "Agent": "REINFORCE"}, {"env_step": 3225600, "rew": 393.1369, "rew_std": 62.1013, "Agent": "REINFORCE"}, {"env_step": 3256320, "rew": 407.482, "rew_std": 51.7877, "Agent": "REINFORCE"}, {"env_step": 3287040, "rew": 400.4091, "rew_std": 62.9306, "Agent": "REINFORCE"}, {"env_step": 3317760, "rew": 393.4948, "rew_std": 64.1455, "Agent": "REINFORCE"}, {"env_step": 3348480, "rew": 406.3127, "rew_std": 56.6238, "Agent": "REINFORCE"}, {"env_step": 3379200, "rew": 397.3756, "rew_std": 64.2524, "Agent": "REINFORCE"}, {"env_step": 3409920, "rew": 415.5537, "rew_std": 78.358, "Agent": "REINFORCE"}, {"env_step": 3440640, "rew": 390.5978, "rew_std": 48.545, "Agent": "REINFORCE"}, {"env_step": 3471360, "rew": 381.6533, "rew_std": 50.4349, "Agent": "REINFORCE"}, {"env_step": 3502080, "rew": 414.7084, "rew_std": 75.1391, "Agent": "REINFORCE"}, {"env_step": 3532800, "rew": 411.7969, "rew_std": 61.9639, "Agent": "REINFORCE"}, {"env_step": 3563520, "rew": 395.862, "rew_std": 51.1378, "Agent": "REINFORCE"}, {"env_step": 3594240, "rew": 394.9354, "rew_std": 72.9375, "Agent": "REINFORCE"}, {"env_step": 3624960, "rew": 389.1405, "rew_std": 57.53, "Agent": "REINFORCE"}, {"env_step": 3655680, "rew": 397.4184, "rew_std": 69.5013, "Agent": "REINFORCE"}, {"env_step": 3686400, "rew": 402.134, "rew_std": 68.5505, "Agent": "REINFORCE"}, {"env_step": 3717120, "rew": 388.0043, "rew_std": 60.6637, "Agent": "REINFORCE"}, {"env_step": 3747840, "rew": 388.857, "rew_std": 61.1726, "Agent": "REINFORCE"}, {"env_step": 3778560, "rew": 395.8664, "rew_std": 80.3261, "Agent": "REINFORCE"}, {"env_step": 3809280, "rew": 391.2895, "rew_std": 50.6701, "Agent": "REINFORCE"}, {"env_step": 3840000, "rew": 375.5112, "rew_std": 51.7993, "Agent": "REINFORCE"}, {"env_step": 3870720, "rew": 385.1477, "rew_std": 74.1886, "Agent": "REINFORCE"}, {"env_step": 3901440, "rew": 385.6211, "rew_std": 58.569, "Agent": "REINFORCE"}, {"env_step": 3932160, "rew": 403.1113, "rew_std": 56.767, "Agent": "REINFORCE"}, {"env_step": 3962880, "rew": 393.6033, "rew_std": 51.2387, "Agent": "REINFORCE"}, {"env_step": 3993600, "rew": 398.7294, "rew_std": 53.0198, "Agent": "REINFORCE"}, {"env_step": 4024320, "rew": 404.4069, "rew_std": 53.2228, "Agent": "REINFORCE"}, {"env_step": 4055040, "rew": 379.2343, "rew_std": 57.0353, "Agent": "REINFORCE"}, {"env_step": 4085760, "rew": 387.5677, "rew_std": 60.9631, "Agent": "REINFORCE"}, {"env_step": 4116480, "rew": 377.398, "rew_std": 49.0542, "Agent": "REINFORCE"}, {"env_step": 4147200, "rew": 406.3338, "rew_std": 63.9793, "Agent": "REINFORCE"}, {"env_step": 4177920, "rew": 405.1044, "rew_std": 58.2787, "Agent": "REINFORCE"}, {"env_step": 4208640, "rew": 394.9841, "rew_std": 57.9348, "Agent": "REINFORCE"}, {"env_step": 4239360, "rew": 425.3993, "rew_std": 55.3748, "Agent": "REINFORCE"}, {"env_step": 4270080, "rew": 395.7237, "rew_std": 50.2733, "Agent": "REINFORCE"}, {"env_step": 4300800, "rew": 410.0695, "rew_std": 58.4723, "Agent": "REINFORCE"}, {"env_step": 4331520, "rew": 420.7224, "rew_std": 54.1291, "Agent": "REINFORCE"}, {"env_step": 4362240, "rew": 407.6703, "rew_std": 57.0062, "Agent": "REINFORCE"}, {"env_step": 4392960, "rew": 388.5928, "rew_std": 59.4505, "Agent": "REINFORCE"}, {"env_step": 4423680, "rew": 398.4204, "rew_std": 38.8942, "Agent": "REINFORCE"}, {"env_step": 4454400, "rew": 422.9708, "rew_std": 62.4621, "Agent": "REINFORCE"}, {"env_step": 4485120, "rew": 407.7592, "rew_std": 45.2911, "Agent": "REINFORCE"}, {"env_step": 4515840, "rew": 397.7424, "rew_std": 64.5239, "Agent": "REINFORCE"}, {"env_step": 4546560, "rew": 364.0651, "rew_std": 42.8236, "Agent": "REINFORCE"}, {"env_step": 4577280, "rew": 387.8125, "rew_std": 67.7828, "Agent": "REINFORCE"}, {"env_step": 4608000, "rew": 387.8849, "rew_std": 56.7927, "Agent": "REINFORCE"}, {"env_step": 4638720, "rew": 395.0737, "rew_std": 52.0062, "Agent": "REINFORCE"}, {"env_step": 4669440, "rew": 377.2858, "rew_std": 39.734, "Agent": "REINFORCE"}, {"env_step": 4700160, "rew": 408.0436, "rew_std": 84.4176, "Agent": "REINFORCE"}, {"env_step": 4730880, "rew": 393.7722, "rew_std": 55.7614, "Agent": "REINFORCE"}, {"env_step": 4761600, "rew": 386.8071, "rew_std": 64.2477, "Agent": "REINFORCE"}, {"env_step": 4792320, "rew": 394.8259, "rew_std": 72.5374, "Agent": "REINFORCE"}, {"env_step": 4823040, "rew": 390.2458, "rew_std": 65.5436, "Agent": "REINFORCE"}, {"env_step": 4853760, "rew": 382.4817, "rew_std": 62.6421, "Agent": "REINFORCE"}, {"env_step": 4884480, "rew": 399.1333, "rew_std": 66.9132, "Agent": "REINFORCE"}, {"env_step": 4915200, "rew": 395.0744, "rew_std": 65.0951, "Agent": "REINFORCE"}, {"env_step": 4945920, "rew": 401.2925, "rew_std": 70.9549, "Agent": "REINFORCE"}, {"env_step": 4976640, "rew": 399.4059, "rew_std": 60.6538, "Agent": "REINFORCE"}, {"env_step": 5007360, "rew": 406.6179, "rew_std": 68.7601, "Agent": "REINFORCE"}, {"env_step": 5038080, "rew": 387.8888, "rew_std": 71.1222, "Agent": "REINFORCE"}, {"env_step": 5068800, "rew": 417.4409, "rew_std": 72.5182, "Agent": "REINFORCE"}, {"env_step": 5099520, "rew": 426.0181, "rew_std": 84.6085, "Agent": "REINFORCE"}, {"env_step": 5130240, "rew": 416.9954, "rew_std": 63.1179, "Agent": "REINFORCE"}, {"env_step": 5160960, "rew": 441.1564, "rew_std": 79.9664, "Agent": "REINFORCE"}, {"env_step": 5191680, "rew": 433.2006, "rew_std": 75.029, "Agent": "REINFORCE"}, {"env_step": 5222400, "rew": 414.7991, "rew_std": 56.961, "Agent": "REINFORCE"}, {"env_step": 5253120, "rew": 414.7902, "rew_std": 71.8032, "Agent": "REINFORCE"}, {"env_step": 5283840, "rew": 425.6593, "rew_std": 62.7082, "Agent": "REINFORCE"}, {"env_step": 5314560, "rew": 419.4213, "rew_std": 53.4877, "Agent": "REINFORCE"}, {"env_step": 5345280, "rew": 424.5813, "rew_std": 77.928, "Agent": "REINFORCE"}, {"env_step": 5376000, "rew": 403.1615, "rew_std": 76.368, "Agent": "REINFORCE"}, {"env_step": 5406720, "rew": 415.7097, "rew_std": 78.4011, "Agent": "REINFORCE"}, {"env_step": 5437440, "rew": 401.4143, "rew_std": 75.9469, "Agent": "REINFORCE"}, {"env_step": 5468160, "rew": 392.6702, "rew_std": 76.9324, "Agent": "REINFORCE"}, {"env_step": 5498880, "rew": 387.4285, "rew_std": 59.568, "Agent": "REINFORCE"}, {"env_step": 5529600, "rew": 384.1608, "rew_std": 57.6717, "Agent": "REINFORCE"}, {"env_step": 5560320, "rew": 397.3022, "rew_std": 80.0755, "Agent": "REINFORCE"}, {"env_step": 5591040, "rew": 401.4928, "rew_std": 73.1254, "Agent": "REINFORCE"}, {"env_step": 5621760, "rew": 405.6077, "rew_std": 64.9382, "Agent": "REINFORCE"}, {"env_step": 5652480, "rew": 403.2244, "rew_std": 63.7616, "Agent": "REINFORCE"}, {"env_step": 5683200, "rew": 391.268, "rew_std": 54.4662, "Agent": "REINFORCE"}, {"env_step": 5713920, "rew": 379.5553, "rew_std": 61.4862, "Agent": "REINFORCE"}, {"env_step": 5744640, "rew": 380.6899, "rew_std": 54.1494, "Agent": "REINFORCE"}, {"env_step": 5775360, "rew": 395.1274, "rew_std": 40.2667, "Agent": "REINFORCE"}, {"env_step": 5806080, "rew": 395.3674, "rew_std": 46.3753, "Agent": "REINFORCE"}, {"env_step": 5836800, "rew": 407.7103, "rew_std": 64.0663, "Agent": "REINFORCE"}, {"env_step": 5867520, "rew": 411.9753, "rew_std": 66.9703, "Agent": "REINFORCE"}, {"env_step": 5898240, "rew": 424.2286, "rew_std": 69.2321, "Agent": "REINFORCE"}, {"env_step": 5928960, "rew": 414.3711, "rew_std": 52.4139, "Agent": "REINFORCE"}, {"env_step": 5959680, "rew": 411.0705, "rew_std": 41.1486, "Agent": "REINFORCE"}, {"env_step": 5990400, "rew": 412.0928, "rew_std": 68.2498, "Agent": "REINFORCE"}, {"env_step": 6021120, "rew": 435.4422, "rew_std": 67.59, "Agent": "REINFORCE"}, {"env_step": 6051840, "rew": 421.7476, "rew_std": 54.2258, "Agent": "REINFORCE"}, {"env_step": 6082560, "rew": 421.9272, "rew_std": 56.9114, "Agent": "REINFORCE"}, {"env_step": 6113280, "rew": 434.3955, "rew_std": 50.4299, "Agent": "REINFORCE"}, {"env_step": 6144000, "rew": 417.1998, "rew_std": 50.2433, "Agent": "REINFORCE"}, {"env_step": 6174720, "rew": 423.4868, "rew_std": 69.0047, "Agent": "REINFORCE"}, {"env_step": 6205440, "rew": 421.0292, "rew_std": 55.9664, "Agent": "REINFORCE"}, {"env_step": 6236160, "rew": 421.5478, "rew_std": 61.617, "Agent": "REINFORCE"}, {"env_step": 6266880, "rew": 438.953, "rew_std": 73.0025, "Agent": "REINFORCE"}, {"env_step": 6297600, "rew": 417.898, "rew_std": 56.2191, "Agent": "REINFORCE"}, {"env_step": 6328320, "rew": 430.4892, "rew_std": 72.7653, "Agent": "REINFORCE"}, {"env_step": 6359040, "rew": 423.9785, "rew_std": 54.0521, "Agent": "REINFORCE"}, {"env_step": 6389760, "rew": 418.1322, "rew_std": 60.4062, "Agent": "REINFORCE"}, {"env_step": 6420480, "rew": 432.6341, "rew_std": 52.3553, "Agent": "REINFORCE"}, {"env_step": 6451200, "rew": 429.442, "rew_std": 66.8584, "Agent": "REINFORCE"}, {"env_step": 6481920, "rew": 429.4091, "rew_std": 55.5702, "Agent": "REINFORCE"}, {"env_step": 6512640, "rew": 433.1991, "rew_std": 57.5528, "Agent": "REINFORCE"}, {"env_step": 6543360, "rew": 421.4787, "rew_std": 64.1868, "Agent": "REINFORCE"}, {"env_step": 6574080, "rew": 429.2496, "rew_std": 68.345, "Agent": "REINFORCE"}, {"env_step": 6604800, "rew": 428.7888, "rew_std": 72.7331, "Agent": "REINFORCE"}, {"env_step": 6635520, "rew": 421.8156, "rew_std": 53.2816, "Agent": "REINFORCE"}, {"env_step": 6666240, "rew": 441.9151, "rew_std": 58.0016, "Agent": "REINFORCE"}, {"env_step": 6696960, "rew": 432.4622, "rew_std": 52.1868, "Agent": "REINFORCE"}, {"env_step": 6727680, "rew": 428.4346, "rew_std": 51.4958, "Agent": "REINFORCE"}, {"env_step": 6758400, "rew": 448.5581, "rew_std": 61.1799, "Agent": "REINFORCE"}, {"env_step": 6789120, "rew": 436.737, "rew_std": 56.055, "Agent": "REINFORCE"}, {"env_step": 6819840, "rew": 450.0283, "rew_std": 60.1757, "Agent": "REINFORCE"}, {"env_step": 6850560, "rew": 426.9679, "rew_std": 53.4073, "Agent": "REINFORCE"}, {"env_step": 6881280, "rew": 445.6709, "rew_std": 58.9099, "Agent": "REINFORCE"}, {"env_step": 6912000, "rew": 435.5077, "rew_std": 50.8351, "Agent": "REINFORCE"}, {"env_step": 6942720, "rew": 438.1268, "rew_std": 47.7346, "Agent": "REINFORCE"}, {"env_step": 6973440, "rew": 435.6654, "rew_std": 48.6435, "Agent": "REINFORCE"}, {"env_step": 7004160, "rew": 447.6092, "rew_std": 45.757, "Agent": "REINFORCE"}, {"env_step": 7034880, "rew": 432.7806, "rew_std": 59.0931, "Agent": "REINFORCE"}, {"env_step": 7065600, "rew": 464.3061, "rew_std": 58.4102, "Agent": "REINFORCE"}, {"env_step": 7096320, "rew": 448.0351, "rew_std": 54.4852, "Agent": "REINFORCE"}, {"env_step": 7127040, "rew": 438.2387, "rew_std": 51.9289, "Agent": "REINFORCE"}, {"env_step": 7157760, "rew": 445.8886, "rew_std": 49.4097, "Agent": "REINFORCE"}, {"env_step": 7188480, "rew": 428.4683, "rew_std": 62.5219, "Agent": "REINFORCE"}, {"env_step": 7219200, "rew": 447.5914, "rew_std": 62.3059, "Agent": "REINFORCE"}, {"env_step": 7249920, "rew": 422.7578, "rew_std": 64.5611, "Agent": "REINFORCE"}, {"env_step": 7280640, "rew": 435.1532, "rew_std": 53.3779, "Agent": "REINFORCE"}, {"env_step": 7311360, "rew": 425.5023, "rew_std": 46.7849, "Agent": "REINFORCE"}, {"env_step": 7342080, "rew": 432.1564, "rew_std": 57.7707, "Agent": "REINFORCE"}, {"env_step": 7372800, "rew": 415.4793, "rew_std": 49.0723, "Agent": "REINFORCE"}, {"env_step": 7403520, "rew": 423.3059, "rew_std": 54.1695, "Agent": "REINFORCE"}, {"env_step": 7434240, "rew": 434.0937, "rew_std": 65.467, "Agent": "REINFORCE"}, {"env_step": 7464960, "rew": 421.1951, "rew_std": 37.9896, "Agent": "REINFORCE"}, {"env_step": 7495680, "rew": 424.2354, "rew_std": 32.1225, "Agent": "REINFORCE"}, {"env_step": 7526400, "rew": 414.4942, "rew_std": 61.44, "Agent": "REINFORCE"}, {"env_step": 7557120, "rew": 420.6777, "rew_std": 48.6498, "Agent": "REINFORCE"}, {"env_step": 7587840, "rew": 421.6311, "rew_std": 41.9637, "Agent": "REINFORCE"}, {"env_step": 7618560, "rew": 419.5132, "rew_std": 44.5098, "Agent": "REINFORCE"}, {"env_step": 7649280, "rew": 453.7393, "rew_std": 72.3884, "Agent": "REINFORCE"}, {"env_step": 7680000, "rew": 431.989, "rew_std": 67.0335, "Agent": "REINFORCE"}, {"env_step": 7710720, "rew": 437.4101, "rew_std": 66.4708, "Agent": "REINFORCE"}, {"env_step": 7741440, "rew": 428.4059, "rew_std": 57.9871, "Agent": "REINFORCE"}, {"env_step": 7772160, "rew": 429.2049, "rew_std": 46.1541, "Agent": "REINFORCE"}, {"env_step": 7802880, "rew": 427.6781, "rew_std": 50.5812, "Agent": "REINFORCE"}, {"env_step": 7833600, "rew": 435.1144, "rew_std": 41.9573, "Agent": "REINFORCE"}, {"env_step": 7864320, "rew": 442.6668, "rew_std": 63.9423, "Agent": "REINFORCE"}, {"env_step": 7895040, "rew": 416.4572, "rew_std": 35.7496, "Agent": "REINFORCE"}, {"env_step": 7925760, "rew": 432.7108, "rew_std": 52.7162, "Agent": "REINFORCE"}, {"env_step": 7956480, "rew": 435.7087, "rew_std": 63.0129, "Agent": "REINFORCE"}, {"env_step": 7987200, "rew": 443.6482, "rew_std": 64.473, "Agent": "REINFORCE"}, {"env_step": 8017920, "rew": 426.112, "rew_std": 65.1428, "Agent": "REINFORCE"}, {"env_step": 8048640, "rew": 419.717, "rew_std": 49.5256, "Agent": "REINFORCE"}, {"env_step": 8079360, "rew": 427.3948, "rew_std": 51.7217, "Agent": "REINFORCE"}, {"env_step": 8110080, "rew": 418.2876, "rew_std": 55.8575, "Agent": "REINFORCE"}, {"env_step": 8140800, "rew": 425.3464, "rew_std": 63.9193, "Agent": "REINFORCE"}, {"env_step": 8171520, "rew": 422.2139, "rew_std": 64.3602, "Agent": "REINFORCE"}, {"env_step": 8202240, "rew": 425.3817, "rew_std": 62.3121, "Agent": "REINFORCE"}, {"env_step": 8232960, "rew": 430.0611, "rew_std": 74.026, "Agent": "REINFORCE"}, {"env_step": 8263680, "rew": 426.6266, "rew_std": 54.1855, "Agent": "REINFORCE"}, {"env_step": 8294400, "rew": 425.9563, "rew_std": 54.4198, "Agent": "REINFORCE"}, {"env_step": 8325120, "rew": 416.8308, "rew_std": 43.4318, "Agent": "REINFORCE"}, {"env_step": 8355840, "rew": 427.924, "rew_std": 53.4286, "Agent": "REINFORCE"}, {"env_step": 8386560, "rew": 425.8094, "rew_std": 48.7615, "Agent": "REINFORCE"}, {"env_step": 8417280, "rew": 436.37, "rew_std": 51.3194, "Agent": "REINFORCE"}, {"env_step": 8448000, "rew": 450.7627, "rew_std": 73.5331, "Agent": "REINFORCE"}, {"env_step": 8478720, "rew": 428.6428, "rew_std": 39.332, "Agent": "REINFORCE"}, {"env_step": 8509440, "rew": 415.792, "rew_std": 35.9552, "Agent": "REINFORCE"}, {"env_step": 8540160, "rew": 442.3338, "rew_std": 55.9098, "Agent": "REINFORCE"}, {"env_step": 8570880, "rew": 420.0383, "rew_std": 40.6303, "Agent": "REINFORCE"}, {"env_step": 8601600, "rew": 427.539, "rew_std": 48.1522, "Agent": "REINFORCE"}, {"env_step": 8632320, "rew": 446.8737, "rew_std": 53.3549, "Agent": "REINFORCE"}, {"env_step": 8663040, "rew": 422.7955, "rew_std": 49.8506, "Agent": "REINFORCE"}, {"env_step": 8693760, "rew": 415.8156, "rew_std": 53.8267, "Agent": "REINFORCE"}, {"env_step": 8724480, "rew": 437.5864, "rew_std": 68.5622, "Agent": "REINFORCE"}, {"env_step": 8755200, "rew": 441.8165, "rew_std": 61.7589, "Agent": "REINFORCE"}, {"env_step": 8785920, "rew": 432.717, "rew_std": 62.6678, "Agent": "REINFORCE"}, {"env_step": 8816640, "rew": 430.1473, "rew_std": 52.4954, "Agent": "REINFORCE"}, {"env_step": 8847360, "rew": 434.8537, "rew_std": 58.7462, "Agent": "REINFORCE"}, {"env_step": 8878080, "rew": 422.1588, "rew_std": 50.1868, "Agent": "REINFORCE"}, {"env_step": 8908800, "rew": 419.7322, "rew_std": 43.2847, "Agent": "REINFORCE"}, {"env_step": 8939520, "rew": 415.9212, "rew_std": 43.222, "Agent": "REINFORCE"}, {"env_step": 8970240, "rew": 425.6137, "rew_std": 60.322, "Agent": "REINFORCE"}, {"env_step": 9000960, "rew": 421.9776, "rew_std": 63.0022, "Agent": "REINFORCE"}, {"env_step": 9031680, "rew": 425.4186, "rew_std": 55.3349, "Agent": "REINFORCE"}, {"env_step": 9062400, "rew": 432.6951, "rew_std": 54.8226, "Agent": "REINFORCE"}, {"env_step": 9093120, "rew": 422.8864, "rew_std": 48.8194, "Agent": "REINFORCE"}, {"env_step": 9123840, "rew": 421.418, "rew_std": 46.865, "Agent": "REINFORCE"}, {"env_step": 9154560, "rew": 433.4248, "rew_std": 42.999, "Agent": "REINFORCE"}, {"env_step": 9185280, "rew": 419.0434, "rew_std": 44.146, "Agent": "REINFORCE"}, {"env_step": 9216000, "rew": 415.736, "rew_std": 41.3839, "Agent": "REINFORCE"}, {"env_step": 9246720, "rew": 414.7147, "rew_std": 36.7896, "Agent": "REINFORCE"}, {"env_step": 9277440, "rew": 434.3581, "rew_std": 54.3064, "Agent": "REINFORCE"}, {"env_step": 9308160, "rew": 425.1595, "rew_std": 55.6919, "Agent": "REINFORCE"}, {"env_step": 9338880, "rew": 442.9066, "rew_std": 67.8961, "Agent": "REINFORCE"}, {"env_step": 9369600, "rew": 429.9155, "rew_std": 52.8229, "Agent": "REINFORCE"}, {"env_step": 9400320, "rew": 433.3738, "rew_std": 67.7054, "Agent": "REINFORCE"}, {"env_step": 9431040, "rew": 424.5967, "rew_std": 58.8434, "Agent": "REINFORCE"}, {"env_step": 9461760, "rew": 432.9595, "rew_std": 46.7897, "Agent": "REINFORCE"}, {"env_step": 9492480, "rew": 427.9092, "rew_std": 43.758, "Agent": "REINFORCE"}, {"env_step": 9523200, "rew": 426.2477, "rew_std": 48.9913, "Agent": "REINFORCE"}, {"env_step": 9553920, "rew": 415.2017, "rew_std": 49.432, "Agent": "REINFORCE"}, {"env_step": 9584640, "rew": 438.1663, "rew_std": 57.0268, "Agent": "REINFORCE"}, {"env_step": 9615360, "rew": 445.0344, "rew_std": 56.2595, "Agent": "REINFORCE"}, {"env_step": 9646080, "rew": 431.6359, "rew_std": 52.3226, "Agent": "REINFORCE"}, {"env_step": 9676800, "rew": 428.2467, "rew_std": 53.144, "Agent": "REINFORCE"}, {"env_step": 9707520, "rew": 430.8904, "rew_std": 50.9101, "Agent": "REINFORCE"}, {"env_step": 9738240, "rew": 435.7712, "rew_std": 74.0012, "Agent": "REINFORCE"}, {"env_step": 9768960, "rew": 440.82, "rew_std": 61.388, "Agent": "REINFORCE"}, {"env_step": 9799680, "rew": 434.538, "rew_std": 62.4474, "Agent": "REINFORCE"}, {"env_step": 9830400, "rew": 414.1024, "rew_std": 48.7035, "Agent": "REINFORCE"}, {"env_step": 9861120, "rew": 427.8226, "rew_std": 53.4481, "Agent": "REINFORCE"}, {"env_step": 9891840, "rew": 426.8527, "rew_std": 56.4748, "Agent": "REINFORCE"}, {"env_step": 9922560, "rew": 415.904, "rew_std": 42.7152, "Agent": "REINFORCE"}, {"env_step": 9953280, "rew": 428.6406, "rew_std": 52.1272, "Agent": "REINFORCE"}, {"env_step": 9984000, "rew": 427.6787, "rew_std": 49.7561, "Agent": "REINFORCE"}, {"env_step": 10014720, "rew": 420.2958, "rew_std": 53.3219, "Agent": "REINFORCE"}, {"env_step": 10045440, "rew": 435.3111, "rew_std": 58.5596, "Agent": "REINFORCE"}, {"env_step": 10076160, "rew": 428.058, "rew_std": 52.0583, "Agent": "REINFORCE"}, {"env_step": 10106880, "rew": 430.8424, "rew_std": 53.0753, "Agent": "REINFORCE"}, {"env_step": 10137600, "rew": 434.0074, "rew_std": 58.8144, "Agent": "REINFORCE"}, {"env_step": 10168320, "rew": 427.3266, "rew_std": 67.2677, "Agent": "REINFORCE"}, {"env_step": 10199040, "rew": 436.6506, "rew_std": 58.1863, "Agent": "REINFORCE"}, {"env_step": 10229760, "rew": 425.308, "rew_std": 53.5519, "Agent": "REINFORCE"}, {"env_step": 0, "rew": 130.7126, "rew_std": 15.6136, "Agent": "A2C"}, {"env_step": 30000, "rew": 353.9919, "rew_std": 44.5764, "Agent": "A2C"}, {"env_step": 60000, "rew": 383.1186, "rew_std": 37.3552, "Agent": "A2C"}, {"env_step": 90000, "rew": 426.6209, "rew_std": 35.6024, "Agent": "A2C"}, {"env_step": 120000, "rew": 430.859, "rew_std": 44.2118, "Agent": "A2C"}, {"env_step": 150000, "rew": 467.4777, "rew_std": 41.7401, "Agent": "A2C"}, {"env_step": 180000, "rew": 520.8538, "rew_std": 68.7925, "Agent": "A2C"}, {"env_step": 210000, "rew": 500.8319, "rew_std": 83.603, "Agent": "A2C"}, {"env_step": 240000, "rew": 496.1667, "rew_std": 39.6322, "Agent": "A2C"}, {"env_step": 270000, "rew": 531.1016, "rew_std": 50.328, "Agent": "A2C"}, {"env_step": 300000, "rew": 536.9946, "rew_std": 36.1136, "Agent": "A2C"}, {"env_step": 330000, "rew": 528.5104, "rew_std": 50.3768, "Agent": "A2C"}, {"env_step": 360000, "rew": 559.3917, "rew_std": 45.3429, "Agent": "A2C"}, {"env_step": 390000, "rew": 525.345, "rew_std": 56.0154, "Agent": "A2C"}, {"env_step": 420000, "rew": 549.3972, "rew_std": 47.0919, "Agent": "A2C"}, {"env_step": 450000, "rew": 551.0984, "rew_std": 50.0343, "Agent": "A2C"}, {"env_step": 480000, "rew": 562.8467, "rew_std": 37.7813, "Agent": "A2C"}, {"env_step": 510000, "rew": 554.9192, "rew_std": 37.7401, "Agent": "A2C"}, {"env_step": 540000, "rew": 613.7179, "rew_std": 109.5216, "Agent": "A2C"}, {"env_step": 570000, "rew": 614.4935, "rew_std": 92.6857, "Agent": "A2C"}, {"env_step": 600000, "rew": 613.4367, "rew_std": 105.5949, "Agent": "A2C"}, {"env_step": 630000, "rew": 671.3968, "rew_std": 117.3872, "Agent": "A2C"}, {"env_step": 660000, "rew": 635.6361, "rew_std": 118.8407, "Agent": "A2C"}, {"env_step": 690000, "rew": 706.9048, "rew_std": 174.3268, "Agent": "A2C"}, {"env_step": 720000, "rew": 855.8826, "rew_std": 337.6497, "Agent": "A2C"}, {"env_step": 750000, "rew": 1007.2058, "rew_std": 564.3479, "Agent": "A2C"}, {"env_step": 780000, "rew": 1003.8926, "rew_std": 625.1997, "Agent": "A2C"}, {"env_step": 810000, "rew": 1359.2944, "rew_std": 909.4713, "Agent": "A2C"}, {"env_step": 840000, "rew": 1214.6042, "rew_std": 851.1437, "Agent": "A2C"}, {"env_step": 870000, "rew": 1349.1214, "rew_std": 731.7768, "Agent": "A2C"}, {"env_step": 900000, "rew": 1412.6869, "rew_std": 851.1827, "Agent": "A2C"}, {"env_step": 930000, "rew": 1726.0036, "rew_std": 1070.0676, "Agent": "A2C"}, {"env_step": 960000, "rew": 1331.9998, "rew_std": 718.4679, "Agent": "A2C"}, {"env_step": 990000, "rew": 1765.9037, "rew_std": 1235.2031, "Agent": "A2C"}, {"env_step": 1020000, "rew": 2476.1129, "rew_std": 1305.1744, "Agent": "A2C"}, {"env_step": 1050000, "rew": 2258.8718, "rew_std": 1242.0474, "Agent": "A2C"}, {"env_step": 1080000, "rew": 2349.4063, "rew_std": 1315.8576, "Agent": "A2C"}, {"env_step": 1110000, "rew": 2802.3667, "rew_std": 1370.2395, "Agent": "A2C"}, {"env_step": 1140000, "rew": 2604.3733, "rew_std": 1407.5554, "Agent": "A2C"}, {"env_step": 1170000, "rew": 2812.3223, "rew_std": 1597.652, "Agent": "A2C"}, {"env_step": 1200000, "rew": 2826.2653, "rew_std": 1363.381, "Agent": "A2C"}, {"env_step": 1230000, "rew": 2606.3455, "rew_std": 1404.4811, "Agent": "A2C"}, {"env_step": 1260000, "rew": 3003.1294, "rew_std": 1444.6226, "Agent": "A2C"}, {"env_step": 1290000, "rew": 2607.4257, "rew_std": 1319.1591, "Agent": "A2C"}, {"env_step": 1320000, "rew": 3091.2105, "rew_std": 1413.2513, "Agent": "A2C"}, {"env_step": 1350000, "rew": 3144.1315, "rew_std": 1492.1207, "Agent": "A2C"}, {"env_step": 1380000, "rew": 3326.8287, "rew_std": 1568.8489, "Agent": "A2C"}, {"env_step": 1410000, "rew": 3587.6421, "rew_std": 1439.4819, "Agent": "A2C"}, {"env_step": 1440000, "rew": 2980.8724, "rew_std": 1454.9531, "Agent": "A2C"}, {"env_step": 1470000, "rew": 3583.043, "rew_std": 1493.8679, "Agent": "A2C"}, {"env_step": 1500000, "rew": 3462.9114, "rew_std": 1451.1828, "Agent": "A2C"}, {"env_step": 1530000, "rew": 3472.9946, "rew_std": 1540.7852, "Agent": "A2C"}, {"env_step": 1560000, "rew": 2653.2359, "rew_std": 1141.8654, "Agent": "A2C"}, {"env_step": 1590000, "rew": 3698.7924, "rew_std": 1472.0893, "Agent": "A2C"}, {"env_step": 1620000, "rew": 3369.2388, "rew_std": 1185.2593, "Agent": "A2C"}, {"env_step": 1650000, "rew": 3751.0162, "rew_std": 1372.8344, "Agent": "A2C"}, {"env_step": 1680000, "rew": 3460.9345, "rew_std": 1359.9235, "Agent": "A2C"}, {"env_step": 1710000, "rew": 4017.4043, "rew_std": 1516.6362, "Agent": "A2C"}, {"env_step": 1740000, "rew": 3841.3238, "rew_std": 1169.35, "Agent": "A2C"}, {"env_step": 1770000, "rew": 4317.4256, "rew_std": 1351.0404, "Agent": "A2C"}, {"env_step": 1800000, "rew": 3615.3238, "rew_std": 1324.6659, "Agent": "A2C"}, {"env_step": 1830000, "rew": 3990.3708, "rew_std": 1115.5551, "Agent": "A2C"}, {"env_step": 1860000, "rew": 4051.3542, "rew_std": 999.8893, "Agent": "A2C"}, {"env_step": 1890000, "rew": 4374.4812, "rew_std": 1537.1536, "Agent": "A2C"}, {"env_step": 1920000, "rew": 4263.77, "rew_std": 1002.1095, "Agent": "A2C"}, {"env_step": 1950000, "rew": 4083.6131, "rew_std": 1265.3289, "Agent": "A2C"}, {"env_step": 1980000, "rew": 4533.5691, "rew_std": 903.9607, "Agent": "A2C"}, {"env_step": 2010000, "rew": 4359.9509, "rew_std": 1059.9943, "Agent": "A2C"}, {"env_step": 2040000, "rew": 4260.8664, "rew_std": 1053.3046, "Agent": "A2C"}, {"env_step": 2070000, "rew": 3934.3529, "rew_std": 1003.8132, "Agent": "A2C"}, {"env_step": 2100000, "rew": 4516.3735, "rew_std": 679.4857, "Agent": "A2C"}, {"env_step": 2130000, "rew": 4899.1379, "rew_std": 650.4817, "Agent": "A2C"}, {"env_step": 2160000, "rew": 4413.6033, "rew_std": 580.1522, "Agent": "A2C"}, {"env_step": 2190000, "rew": 4688.5056, "rew_std": 570.3642, "Agent": "A2C"}, {"env_step": 2220000, "rew": 4468.6075, "rew_std": 710.3848, "Agent": "A2C"}, {"env_step": 2250000, "rew": 4835.1334, "rew_std": 760.7642, "Agent": "A2C"}, {"env_step": 2280000, "rew": 4374.0217, "rew_std": 776.1664, "Agent": "A2C"}, {"env_step": 2310000, "rew": 4442.3485, "rew_std": 710.9176, "Agent": "A2C"}, {"env_step": 2340000, "rew": 4771.5979, "rew_std": 737.0396, "Agent": "A2C"}, {"env_step": 2370000, "rew": 4354.7658, "rew_std": 719.8449, "Agent": "A2C"}, {"env_step": 2400000, "rew": 4510.2853, "rew_std": 732.1358, "Agent": "A2C"}, {"env_step": 2430000, "rew": 4661.3797, "rew_std": 887.0676, "Agent": "A2C"}, {"env_step": 2460000, "rew": 4849.6007, "rew_std": 729.8122, "Agent": "A2C"}, {"env_step": 2490000, "rew": 4860.8034, "rew_std": 752.8378, "Agent": "A2C"}, {"env_step": 2520000, "rew": 5118.544, "rew_std": 512.6474, "Agent": "A2C"}, {"env_step": 2550000, "rew": 5234.9514, "rew_std": 749.6186, "Agent": "A2C"}, {"env_step": 2580000, "rew": 5014.9994, "rew_std": 851.1116, "Agent": "A2C"}, {"env_step": 2610000, "rew": 4910.4339, "rew_std": 593.0186, "Agent": "A2C"}, {"env_step": 2640000, "rew": 5285.795, "rew_std": 588.8437, "Agent": "A2C"}, {"env_step": 2670000, "rew": 5061.3064, "rew_std": 570.5551, "Agent": "A2C"}, {"env_step": 2700000, "rew": 5231.117, "rew_std": 574.5865, "Agent": "A2C"}, {"env_step": 2730000, "rew": 5001.9283, "rew_std": 701.5343, "Agent": "A2C"}, {"env_step": 2760000, "rew": 4992.397, "rew_std": 500.2804, "Agent": "A2C"}, {"env_step": 2790000, "rew": 5147.4111, "rew_std": 510.2497, "Agent": "A2C"}, {"env_step": 2820000, "rew": 4734.8069, "rew_std": 661.113, "Agent": "A2C"}, {"env_step": 2850000, "rew": 5067.7192, "rew_std": 581.2196, "Agent": "A2C"}, {"env_step": 2880000, "rew": 5300.8283, "rew_std": 668.5736, "Agent": "A2C"}, {"env_step": 2910000, "rew": 5316.5694, "rew_std": 554.7586, "Agent": "A2C"}, {"env_step": 2940000, "rew": 5112.4173, "rew_std": 489.4034, "Agent": "A2C"}, {"env_step": 2970000, "rew": 4819.7192, "rew_std": 603.6194, "Agent": "A2C"}, {"env_step": 3000000, "rew": 5138.2266, "rew_std": 458.5997, "Agent": "A2C"}] \ No newline at end of file diff --git a/examples/mujoco/benchmark/InvertedDoublePendulum-v2/result.json b/examples/mujoco/benchmark/InvertedDoublePendulum-v2/result.json new file mode 100644 index 0000000..c76ab70 --- /dev/null +++ b/examples/mujoco/benchmark/InvertedDoublePendulum-v2/result.json @@ -0,0 +1 @@ +[{"env_step": 0, "rew": 87.8507, "rew_std": 12.8325, "Agent": "SAC"}, {"env_step": 5000, "rew": 756.2125, "rew_std": 257.9012, "Agent": "SAC"}, {"env_step": 10000, "rew": 7728.6778, "rew_std": 3036.0068, "Agent": "SAC"}, {"env_step": 15000, "rew": 9345.2064, "rew_std": 7.1745, "Agent": "SAC"}, {"env_step": 20000, "rew": 9067.9081, "rew_std": 568.0202, "Agent": "SAC"}, {"env_step": 25000, "rew": 9251.3634, "rew_std": 273.4882, "Agent": "SAC"}, {"env_step": 30000, "rew": 9339.7022, "rew_std": 13.0168, "Agent": "SAC"}, {"env_step": 35000, "rew": 9341.0825, "rew_std": 8.3511, "Agent": "SAC"}, {"env_step": 40000, "rew": 9179.9443, "rew_std": 482.8777, "Agent": "SAC"}, {"env_step": 45000, "rew": 9349.2818, "rew_std": 7.345, "Agent": "SAC"}, {"env_step": 50000, "rew": 9347.1812, "rew_std": 5.5747, "Agent": "SAC"}, {"env_step": 55000, "rew": 9343.8767, "rew_std": 7.8371, "Agent": "SAC"}, {"env_step": 60000, "rew": 9348.0068, "rew_std": 5.8836, "Agent": "SAC"}, {"env_step": 65000, "rew": 9076.2041, "rew_std": 410.8547, "Agent": "SAC"}, {"env_step": 70000, "rew": 9351.7263, "rew_std": 4.4372, "Agent": "SAC"}, {"env_step": 75000, "rew": 9353.7934, "rew_std": 3.4108, "Agent": "SAC"}, {"env_step": 80000, "rew": 9353.4646, "rew_std": 3.3222, "Agent": "SAC"}, {"env_step": 85000, "rew": 9355.8036, "rew_std": 2.1557, "Agent": "SAC"}, {"env_step": 90000, "rew": 9353.5623, "rew_std": 4.9767, "Agent": "SAC"}, {"env_step": 95000, "rew": 9353.3058, "rew_std": 4.9776, "Agent": "SAC"}, {"env_step": 100000, "rew": 9354.1422, "rew_std": 4.1689, "Agent": "SAC"}, {"env_step": 105000, "rew": 9353.5129, "rew_std": 4.658, "Agent": "SAC"}, {"env_step": 110000, "rew": 9355.5423, "rew_std": 2.9517, "Agent": "SAC"}, {"env_step": 115000, "rew": 9193.4331, "rew_std": 492.2324, "Agent": "SAC"}, {"env_step": 120000, "rew": 9356.9442, "rew_std": 4.6735, "Agent": "SAC"}, {"env_step": 125000, "rew": 9358.2736, "rew_std": 1.8483, "Agent": "SAC"}, {"env_step": 130000, "rew": 9357.66, "rew_std": 2.5569, "Agent": "SAC"}, {"env_step": 135000, "rew": 9357.4056, "rew_std": 2.5524, "Agent": "SAC"}, {"env_step": 140000, "rew": 9357.8777, "rew_std": 2.6114, "Agent": "SAC"}, {"env_step": 145000, "rew": 9357.2298, "rew_std": 3.1278, "Agent": "SAC"}, {"env_step": 150000, "rew": 8600.7654, "rew_std": 2274.1217, "Agent": "SAC"}, {"env_step": 155000, "rew": 9199.7953, "rew_std": 472.8582, "Agent": "SAC"}, {"env_step": 160000, "rew": 9357.0128, "rew_std": 2.0788, "Agent": "SAC"}, {"env_step": 165000, "rew": 9270.2483, "rew_std": 266.3928, "Agent": "SAC"}, {"env_step": 170000, "rew": 9359.2144, "rew_std": 0.4645, "Agent": "SAC"}, {"env_step": 175000, "rew": 9358.4945, "rew_std": 1.9754, "Agent": "SAC"}, {"env_step": 180000, "rew": 9358.0269, "rew_std": 2.3514, "Agent": "SAC"}, {"env_step": 185000, "rew": 9358.1878, "rew_std": 2.0733, "Agent": "SAC"}, {"env_step": 190000, "rew": 9359.2768, "rew_std": 0.4847, "Agent": "SAC"}, {"env_step": 195000, "rew": 9359.55, "rew_std": 0.3739, "Agent": "SAC"}, {"env_step": 200000, "rew": 9358.4272, "rew_std": 2.6085, "Agent": "SAC"}, {"env_step": 205000, "rew": 9189.6563, "rew_std": 504.8487, "Agent": "SAC"}, {"env_step": 210000, "rew": 9358.7361, "rew_std": 1.912, "Agent": "SAC"}, {"env_step": 215000, "rew": 9359.223, "rew_std": 0.9374, "Agent": "SAC"}, {"env_step": 220000, "rew": 9358.6182, "rew_std": 1.5412, "Agent": "SAC"}, {"env_step": 225000, "rew": 9357.2911, "rew_std": 3.1816, "Agent": "SAC"}, {"env_step": 230000, "rew": 9358.2075, "rew_std": 1.7019, "Agent": "SAC"}, {"env_step": 235000, "rew": 9358.601, "rew_std": 1.2267, "Agent": "SAC"}, {"env_step": 240000, "rew": 9359.2408, "rew_std": 0.8431, "Agent": "SAC"}, {"env_step": 245000, "rew": 9272.3552, "rew_std": 256.1615, "Agent": "SAC"}, {"env_step": 250000, "rew": 9358.7978, "rew_std": 0.9196, "Agent": "SAC"}, {"env_step": 255000, "rew": 9359.4003, "rew_std": 0.611, "Agent": "SAC"}, {"env_step": 260000, "rew": 9289.9692, "rew_std": 206.5769, "Agent": "SAC"}, {"env_step": 265000, "rew": 9359.103, "rew_std": 0.7612, "Agent": "SAC"}, {"env_step": 270000, "rew": 9357.6548, "rew_std": 2.754, "Agent": "SAC"}, {"env_step": 275000, "rew": 9359.2981, "rew_std": 0.6554, "Agent": "SAC"}, {"env_step": 280000, "rew": 9358.866, "rew_std": 0.9736, "Agent": "SAC"}, {"env_step": 285000, "rew": 9357.7487, "rew_std": 3.921, "Agent": "SAC"}, {"env_step": 290000, "rew": 9357.2292, "rew_std": 2.6113, "Agent": "SAC"}, {"env_step": 295000, "rew": 8815.7244, "rew_std": 1630.2044, "Agent": "SAC"}, {"env_step": 300000, "rew": 8736.6906, "rew_std": 1865.9664, "Agent": "SAC"}, {"env_step": 305000, "rew": 9358.1978, "rew_std": 1.6803, "Agent": "SAC"}, {"env_step": 310000, "rew": 9357.3869, "rew_std": 3.205, "Agent": "SAC"}, {"env_step": 315000, "rew": 8690.9313, "rew_std": 1999.8899, "Agent": "SAC"}, {"env_step": 320000, "rew": 9357.4604, "rew_std": 2.3499, "Agent": "SAC"}, {"env_step": 325000, "rew": 9357.4597, "rew_std": 2.775, "Agent": "SAC"}, {"env_step": 330000, "rew": 9358.9668, "rew_std": 0.7506, "Agent": "SAC"}, {"env_step": 335000, "rew": 9355.8857, "rew_std": 10.1018, "Agent": "SAC"}, {"env_step": 340000, "rew": 8677.3081, "rew_std": 2045.4936, "Agent": "SAC"}, {"env_step": 345000, "rew": 8398.0554, "rew_std": 2619.1756, "Agent": "SAC"}, {"env_step": 350000, "rew": 7765.9506, "rew_std": 3186.7966, "Agent": "SAC"}, {"env_step": 355000, "rew": 9357.3369, "rew_std": 3.3379, "Agent": "SAC"}, {"env_step": 360000, "rew": 9358.0212, "rew_std": 1.5434, "Agent": "SAC"}, {"env_step": 365000, "rew": 9357.3291, "rew_std": 2.4368, "Agent": "SAC"}, {"env_step": 370000, "rew": 8527.6686, "rew_std": 2486.9701, "Agent": "SAC"}, {"env_step": 375000, "rew": 9356.9724, "rew_std": 3.4184, "Agent": "SAC"}, {"env_step": 380000, "rew": 9357.8325, "rew_std": 2.2129, "Agent": "SAC"}, {"env_step": 385000, "rew": 9358.8762, "rew_std": 1.1965, "Agent": "SAC"}, {"env_step": 390000, "rew": 9358.5068, "rew_std": 1.6315, "Agent": "SAC"}, {"env_step": 395000, "rew": 9358.6581, "rew_std": 1.2426, "Agent": "SAC"}, {"env_step": 400000, "rew": 9284.3148, "rew_std": 221.399, "Agent": "SAC"}, {"env_step": 405000, "rew": 8510.3777, "rew_std": 2542.6878, "Agent": "SAC"}, {"env_step": 410000, "rew": 8579.7797, "rew_std": 2329.1843, "Agent": "SAC"}, {"env_step": 415000, "rew": 9357.8998, "rew_std": 2.7608, "Agent": "SAC"}, {"env_step": 420000, "rew": 9357.865, "rew_std": 2.1883, "Agent": "SAC"}, {"env_step": 425000, "rew": 9356.5476, "rew_std": 4.8174, "Agent": "SAC"}, {"env_step": 430000, "rew": 9358.2478, "rew_std": 2.0621, "Agent": "SAC"}, {"env_step": 435000, "rew": 9357.224, "rew_std": 3.8015, "Agent": "SAC"}, {"env_step": 440000, "rew": 9357.3562, "rew_std": 3.3264, "Agent": "SAC"}, {"env_step": 445000, "rew": 9358.0301, "rew_std": 1.9789, "Agent": "SAC"}, {"env_step": 450000, "rew": 9358.1439, "rew_std": 1.3295, "Agent": "SAC"}, {"env_step": 455000, "rew": 9357.0635, "rew_std": 2.7428, "Agent": "SAC"}, {"env_step": 460000, "rew": 9358.3828, "rew_std": 2.1765, "Agent": "SAC"}, {"env_step": 465000, "rew": 9358.1382, "rew_std": 1.4252, "Agent": "SAC"}, {"env_step": 470000, "rew": 9182.7382, "rew_std": 521.5833, "Agent": "SAC"}, {"env_step": 475000, "rew": 8551.3778, "rew_std": 2412.8404, "Agent": "SAC"}, {"env_step": 480000, "rew": 9357.468, "rew_std": 2.7522, "Agent": "SAC"}, {"env_step": 485000, "rew": 9357.9983, "rew_std": 1.7244, "Agent": "SAC"}, {"env_step": 490000, "rew": 8534.6716, "rew_std": 2470.8785, "Agent": "SAC"}, {"env_step": 495000, "rew": 8514.5436, "rew_std": 2528.1433, "Agent": "SAC"}, {"env_step": 500000, "rew": 9296.5499, "rew_std": 183.7219, "Agent": "SAC"}, {"env_step": 505000, "rew": 9357.6859, "rew_std": 1.5885, "Agent": "SAC"}, {"env_step": 510000, "rew": 7917.2816, "rew_std": 2901.6283, "Agent": "SAC"}, {"env_step": 515000, "rew": 9355.8243, "rew_std": 2.8363, "Agent": "SAC"}, {"env_step": 520000, "rew": 8639.4952, "rew_std": 2154.2267, "Agent": "SAC"}, {"env_step": 525000, "rew": 9357.7612, "rew_std": 1.9323, "Agent": "SAC"}, {"env_step": 530000, "rew": 9357.5089, "rew_std": 2.2904, "Agent": "SAC"}, {"env_step": 535000, "rew": 8953.6934, "rew_std": 1213.18, "Agent": "SAC"}, {"env_step": 540000, "rew": 9357.9182, "rew_std": 2.6973, "Agent": "SAC"}, {"env_step": 545000, "rew": 9358.0829, "rew_std": 2.6316, "Agent": "SAC"}, {"env_step": 550000, "rew": 9358.3228, "rew_std": 1.9631, "Agent": "SAC"}, {"env_step": 555000, "rew": 9358.631, "rew_std": 1.5429, "Agent": "SAC"}, {"env_step": 560000, "rew": 9357.9939, "rew_std": 2.6106, "Agent": "SAC"}, {"env_step": 565000, "rew": 9357.863, "rew_std": 2.1863, "Agent": "SAC"}, {"env_step": 570000, "rew": 9269.8532, "rew_std": 264.2835, "Agent": "SAC"}, {"env_step": 575000, "rew": 8532.672, "rew_std": 2478.1488, "Agent": "SAC"}, {"env_step": 580000, "rew": 9268.4563, "rew_std": 269.2364, "Agent": "SAC"}, {"env_step": 585000, "rew": 9356.9148, "rew_std": 2.7939, "Agent": "SAC"}, {"env_step": 590000, "rew": 9357.5543, "rew_std": 3.2443, "Agent": "SAC"}, {"env_step": 595000, "rew": 9357.463, "rew_std": 4.0439, "Agent": "SAC"}, {"env_step": 600000, "rew": 9358.4626, "rew_std": 1.9303, "Agent": "SAC"}, {"env_step": 605000, "rew": 9358.3671, "rew_std": 1.6862, "Agent": "SAC"}, {"env_step": 610000, "rew": 9356.2164, "rew_std": 3.1168, "Agent": "SAC"}, {"env_step": 615000, "rew": 9196.5389, "rew_std": 479.0729, "Agent": "SAC"}, {"env_step": 620000, "rew": 9356.9592, "rew_std": 3.5518, "Agent": "SAC"}, {"env_step": 625000, "rew": 9357.161, "rew_std": 2.5809, "Agent": "SAC"}, {"env_step": 630000, "rew": 9357.3341, "rew_std": 3.0693, "Agent": "SAC"}, {"env_step": 635000, "rew": 9356.3924, "rew_std": 4.6242, "Agent": "SAC"}, {"env_step": 640000, "rew": 9358.5012, "rew_std": 1.1927, "Agent": "SAC"}, {"env_step": 645000, "rew": 9354.8708, "rew_std": 8.4384, "Agent": "SAC"}, {"env_step": 650000, "rew": 9356.4388, "rew_std": 4.5926, "Agent": "SAC"}, {"env_step": 655000, "rew": 9268.8286, "rew_std": 268.5847, "Agent": "SAC"}, {"env_step": 660000, "rew": 9358.5155, "rew_std": 0.8397, "Agent": "SAC"}, {"env_step": 665000, "rew": 9356.109, "rew_std": 4.5035, "Agent": "SAC"}, {"env_step": 670000, "rew": 8668.209, "rew_std": 2067.7502, "Agent": "SAC"}, {"env_step": 675000, "rew": 9356.4476, "rew_std": 2.7218, "Agent": "SAC"}, {"env_step": 680000, "rew": 9356.7041, "rew_std": 2.8959, "Agent": "SAC"}, {"env_step": 685000, "rew": 9183.2542, "rew_std": 507.7661, "Agent": "SAC"}, {"env_step": 690000, "rew": 9185.6077, "rew_std": 512.599, "Agent": "SAC"}, {"env_step": 695000, "rew": 9353.9835, "rew_std": 5.7664, "Agent": "SAC"}, {"env_step": 700000, "rew": 9356.6336, "rew_std": 2.2203, "Agent": "SAC"}, {"env_step": 705000, "rew": 9356.3002, "rew_std": 1.8897, "Agent": "SAC"}, {"env_step": 710000, "rew": 9358.1446, "rew_std": 1.2315, "Agent": "SAC"}, {"env_step": 715000, "rew": 9356.4302, "rew_std": 5.0337, "Agent": "SAC"}, {"env_step": 720000, "rew": 9357.4375, "rew_std": 2.3157, "Agent": "SAC"}, {"env_step": 725000, "rew": 9357.2368, "rew_std": 2.2183, "Agent": "SAC"}, {"env_step": 730000, "rew": 9358.8531, "rew_std": 0.8592, "Agent": "SAC"}, {"env_step": 735000, "rew": 9356.496, "rew_std": 4.2008, "Agent": "SAC"}, {"env_step": 740000, "rew": 9358.0214, "rew_std": 1.9654, "Agent": "SAC"}, {"env_step": 745000, "rew": 9185.1656, "rew_std": 514.708, "Agent": "SAC"}, {"env_step": 750000, "rew": 9357.9964, "rew_std": 1.2584, "Agent": "SAC"}, {"env_step": 755000, "rew": 9185.0382, "rew_std": 512.9892, "Agent": "SAC"}, {"env_step": 760000, "rew": 9355.9867, "rew_std": 4.5162, "Agent": "SAC"}, {"env_step": 765000, "rew": 9357.3264, "rew_std": 2.0687, "Agent": "SAC"}, {"env_step": 770000, "rew": 9358.5004, "rew_std": 1.0138, "Agent": "SAC"}, {"env_step": 775000, "rew": 9268.6565, "rew_std": 255.2189, "Agent": "SAC"}, {"env_step": 780000, "rew": 9358.1214, "rew_std": 1.6113, "Agent": "SAC"}, {"env_step": 785000, "rew": 9356.2192, "rew_std": 3.7295, "Agent": "SAC"}, {"env_step": 790000, "rew": 9356.6503, "rew_std": 3.8859, "Agent": "SAC"}, {"env_step": 795000, "rew": 9357.0889, "rew_std": 3.9538, "Agent": "SAC"}, {"env_step": 800000, "rew": 9357.3384, "rew_std": 1.7422, "Agent": "SAC"}, {"env_step": 805000, "rew": 9356.6001, "rew_std": 2.6288, "Agent": "SAC"}, {"env_step": 810000, "rew": 9355.0373, "rew_std": 4.3317, "Agent": "SAC"}, {"env_step": 815000, "rew": 9272.1791, "rew_std": 244.7629, "Agent": "SAC"}, {"env_step": 820000, "rew": 8584.3171, "rew_std": 2312.6697, "Agent": "SAC"}, {"env_step": 825000, "rew": 9355.9172, "rew_std": 3.118, "Agent": "SAC"}, {"env_step": 830000, "rew": 9355.178, "rew_std": 4.1283, "Agent": "SAC"}, {"env_step": 835000, "rew": 9354.8275, "rew_std": 3.0273, "Agent": "SAC"}, {"env_step": 840000, "rew": 9356.123, "rew_std": 2.7092, "Agent": "SAC"}, {"env_step": 845000, "rew": 9356.9006, "rew_std": 2.6423, "Agent": "SAC"}, {"env_step": 850000, "rew": 9097.4006, "rew_std": 776.0303, "Agent": "SAC"}, {"env_step": 855000, "rew": 9188.9318, "rew_std": 499.1948, "Agent": "SAC"}, {"env_step": 860000, "rew": 9357.2354, "rew_std": 2.9334, "Agent": "SAC"}, {"env_step": 865000, "rew": 9355.8375, "rew_std": 4.2123, "Agent": "SAC"}, {"env_step": 870000, "rew": 9353.5733, "rew_std": 9.3714, "Agent": "SAC"}, {"env_step": 875000, "rew": 9357.4238, "rew_std": 2.3234, "Agent": "SAC"}, {"env_step": 880000, "rew": 9356.2627, "rew_std": 6.0144, "Agent": "SAC"}, {"env_step": 885000, "rew": 9357.9774, "rew_std": 2.1263, "Agent": "SAC"}, {"env_step": 890000, "rew": 9357.6414, "rew_std": 1.6319, "Agent": "SAC"}, {"env_step": 895000, "rew": 9357.9668, "rew_std": 1.9952, "Agent": "SAC"}, {"env_step": 900000, "rew": 8471.5008, "rew_std": 2378.4696, "Agent": "SAC"}, {"env_step": 905000, "rew": 9356.7069, "rew_std": 3.7738, "Agent": "SAC"}, {"env_step": 910000, "rew": 9357.1332, "rew_std": 3.5212, "Agent": "SAC"}, {"env_step": 915000, "rew": 9355.6515, "rew_std": 4.1753, "Agent": "SAC"}, {"env_step": 920000, "rew": 9353.0939, "rew_std": 4.6991, "Agent": "SAC"}, {"env_step": 925000, "rew": 8509.6734, "rew_std": 2544.1191, "Agent": "SAC"}, {"env_step": 930000, "rew": 9356.5265, "rew_std": 3.1623, "Agent": "SAC"}, {"env_step": 935000, "rew": 9356.4495, "rew_std": 2.4185, "Agent": "SAC"}, {"env_step": 940000, "rew": 9356.1637, "rew_std": 3.4761, "Agent": "SAC"}, {"env_step": 945000, "rew": 8557.7586, "rew_std": 2391.4699, "Agent": "SAC"}, {"env_step": 950000, "rew": 9355.9067, "rew_std": 3.3597, "Agent": "SAC"}, {"env_step": 955000, "rew": 9355.8109, "rew_std": 2.0901, "Agent": "SAC"}, {"env_step": 960000, "rew": 9356.5445, "rew_std": 2.6151, "Agent": "SAC"}, {"env_step": 965000, "rew": 9356.6557, "rew_std": 3.2263, "Agent": "SAC"}, {"env_step": 970000, "rew": 9356.6071, "rew_std": 3.243, "Agent": "SAC"}, {"env_step": 975000, "rew": 9355.9793, "rew_std": 3.4355, "Agent": "SAC"}, {"env_step": 980000, "rew": 8523.7853, "rew_std": 2496.1878, "Agent": "SAC"}, {"env_step": 985000, "rew": 9354.4498, "rew_std": 4.182, "Agent": "SAC"}, {"env_step": 990000, "rew": 9357.3811, "rew_std": 1.9945, "Agent": "SAC"}, {"env_step": 995000, "rew": 9357.1444, "rew_std": 1.9249, "Agent": "SAC"}, {"env_step": 1000000, "rew": 9269.378, "rew_std": 265.5736, "Agent": "SAC"}, {"env_step": 1005000, "rew": 9355.054, "rew_std": 5.6732, "Agent": "SAC"}, {"env_step": 1010000, "rew": 8485.4707, "rew_std": 2610.2802, "Agent": "SAC"}, {"env_step": 1015000, "rew": 9355.2511, "rew_std": 3.5887, "Agent": "SAC"}, {"env_step": 1020000, "rew": 9357.6654, "rew_std": 2.0783, "Agent": "SAC"}, {"env_step": 1025000, "rew": 8824.9238, "rew_std": 1321.7067, "Agent": "SAC"}, {"env_step": 1030000, "rew": 9354.8854, "rew_std": 8.5901, "Agent": "SAC"}, {"env_step": 1035000, "rew": 8748.8442, "rew_std": 1823.8367, "Agent": "SAC"}, {"env_step": 1040000, "rew": 9355.839, "rew_std": 3.0407, "Agent": "SAC"}, {"env_step": 1045000, "rew": 9186.054, "rew_std": 498.5165, "Agent": "SAC"}, {"env_step": 1050000, "rew": 9355.953, "rew_std": 4.6684, "Agent": "SAC"}, {"env_step": 1055000, "rew": 8556.4775, "rew_std": 2402.5939, "Agent": "SAC"}, {"env_step": 1060000, "rew": 7979.0473, "rew_std": 2758.1903, "Agent": "SAC"}, {"env_step": 1065000, "rew": 9357.7278, "rew_std": 1.6492, "Agent": "SAC"}, {"env_step": 1070000, "rew": 9357.3245, "rew_std": 1.9565, "Agent": "SAC"}, {"env_step": 1075000, "rew": 9355.7818, "rew_std": 5.2267, "Agent": "SAC"}, {"env_step": 1080000, "rew": 8654.1202, "rew_std": 2108.016, "Agent": "SAC"}, {"env_step": 1085000, "rew": 9264.301, "rew_std": 280.1512, "Agent": "SAC"}, {"env_step": 1090000, "rew": 8669.3311, "rew_std": 1414.0249, "Agent": "SAC"}, {"env_step": 1095000, "rew": 9357.4627, "rew_std": 2.0457, "Agent": "SAC"}, {"env_step": 1100000, "rew": 9356.6594, "rew_std": 2.7395, "Agent": "SAC"}, {"env_step": 1105000, "rew": 9267.9669, "rew_std": 265.7297, "Agent": "SAC"}, {"env_step": 1110000, "rew": 9356.5423, "rew_std": 2.5126, "Agent": "SAC"}, {"env_step": 1115000, "rew": 8597.87, "rew_std": 2277.7467, "Agent": "SAC"}, {"env_step": 1120000, "rew": 9264.0892, "rew_std": 276.7349, "Agent": "SAC"}, {"env_step": 1125000, "rew": 7900.357, "rew_std": 2951.27, "Agent": "SAC"}, {"env_step": 1130000, "rew": 9354.4699, "rew_std": 6.3684, "Agent": "SAC"}, {"env_step": 1135000, "rew": 9356.0906, "rew_std": 4.4826, "Agent": "SAC"}, {"env_step": 1140000, "rew": 9354.2508, "rew_std": 6.1105, "Agent": "SAC"}, {"env_step": 1145000, "rew": 9356.163, "rew_std": 3.1381, "Agent": "SAC"}, {"env_step": 1150000, "rew": 9354.3438, "rew_std": 4.3484, "Agent": "SAC"}, {"env_step": 1155000, "rew": 9353.4378, "rew_std": 6.3306, "Agent": "SAC"}, {"env_step": 1160000, "rew": 9356.8536, "rew_std": 3.0736, "Agent": "SAC"}, {"env_step": 1165000, "rew": 9356.9584, "rew_std": 2.9037, "Agent": "SAC"}, {"env_step": 1170000, "rew": 9263.6426, "rew_std": 279.4552, "Agent": "SAC"}, {"env_step": 1175000, "rew": 9356.6183, "rew_std": 3.1622, "Agent": "SAC"}, {"env_step": 1180000, "rew": 9261.2321, "rew_std": 273.7961, "Agent": "SAC"}, {"env_step": 1185000, "rew": 9355.9645, "rew_std": 3.8095, "Agent": "SAC"}, {"env_step": 1190000, "rew": 9356.537, "rew_std": 2.3578, "Agent": "SAC"}, {"env_step": 1195000, "rew": 9357.0243, "rew_std": 2.3114, "Agent": "SAC"}, {"env_step": 1200000, "rew": 9007.6879, "rew_std": 1041.6742, "Agent": "SAC"}, {"env_step": 1205000, "rew": 9356.6688, "rew_std": 2.3098, "Agent": "SAC"}, {"env_step": 1210000, "rew": 9357.0209, "rew_std": 2.1465, "Agent": "SAC"}, {"env_step": 1215000, "rew": 9356.3172, "rew_std": 2.1724, "Agent": "SAC"}, {"env_step": 1220000, "rew": 9355.0894, "rew_std": 4.8487, "Agent": "SAC"}, {"env_step": 1225000, "rew": 9355.2638, "rew_std": 2.3195, "Agent": "SAC"}, {"env_step": 1230000, "rew": 9355.6464, "rew_std": 2.4159, "Agent": "SAC"}, {"env_step": 1235000, "rew": 9353.5888, "rew_std": 7.0113, "Agent": "SAC"}, {"env_step": 1240000, "rew": 9355.6813, "rew_std": 2.9253, "Agent": "SAC"}, {"env_step": 1245000, "rew": 9356.5849, "rew_std": 2.2778, "Agent": "SAC"}, {"env_step": 1250000, "rew": 9352.2148, "rew_std": 12.6716, "Agent": "SAC"}, {"env_step": 0, "rew": 87.8507, "rew_std": 12.8325, "Agent": "DDPG"}, {"env_step": 5000, "rew": 1148.5608, "rew_std": 1411.8006, "Agent": "DDPG"}, {"env_step": 10000, "rew": 5683.871, "rew_std": 3495.3789, "Agent": "DDPG"}, {"env_step": 15000, "rew": 6564.777, "rew_std": 4061.7699, "Agent": "DDPG"}, {"env_step": 20000, "rew": 7379.0819, "rew_std": 3461.198, "Agent": "DDPG"}, {"env_step": 25000, "rew": 8364.3434, "rew_std": 2778.9295, "Agent": "DDPG"}, {"env_step": 30000, "rew": 8363.9272, "rew_std": 2779.0584, "Agent": "DDPG"}, {"env_step": 35000, "rew": 8354.9727, "rew_std": 2775.5436, "Agent": "DDPG"}, {"env_step": 40000, "rew": 8354.2209, "rew_std": 2775.5533, "Agent": "DDPG"}, {"env_step": 45000, "rew": 8353.3548, "rew_std": 2775.2758, "Agent": "DDPG"}, {"env_step": 50000, "rew": 8346.184, "rew_std": 2773.1462, "Agent": "DDPG"}, {"env_step": 55000, "rew": 8340.5262, "rew_std": 2770.9872, "Agent": "DDPG"}, {"env_step": 60000, "rew": 8338.25, "rew_std": 2770.5018, "Agent": "DDPG"}, {"env_step": 65000, "rew": 8336.6722, "rew_std": 2769.444, "Agent": "DDPG"}, {"env_step": 70000, "rew": 8332.4935, "rew_std": 2767.7795, "Agent": "DDPG"}, {"env_step": 75000, "rew": 8327.6732, "rew_std": 2766.4379, "Agent": "DDPG"}, {"env_step": 80000, "rew": 8300.2123, "rew_std": 2758.217, "Agent": "DDPG"}, {"env_step": 85000, "rew": 8321.4129, "rew_std": 2764.0878, "Agent": "DDPG"}, {"env_step": 90000, "rew": 8315.6078, "rew_std": 2762.7047, "Agent": "DDPG"}, {"env_step": 95000, "rew": 8313.166, "rew_std": 2761.6036, "Agent": "DDPG"}, {"env_step": 100000, "rew": 8312.4057, "rew_std": 2761.3488, "Agent": "DDPG"}, {"env_step": 105000, "rew": 8311.4733, "rew_std": 2761.5601, "Agent": "DDPG"}, {"env_step": 110000, "rew": 8307.8776, "rew_std": 2759.8442, "Agent": "DDPG"}, {"env_step": 115000, "rew": 8306.386, "rew_std": 2759.3404, "Agent": "DDPG"}, {"env_step": 120000, "rew": 8301.8255, "rew_std": 2758.0951, "Agent": "DDPG"}, {"env_step": 125000, "rew": 8301.0734, "rew_std": 2757.3022, "Agent": "DDPG"}, {"env_step": 130000, "rew": 8297.8948, "rew_std": 2756.7646, "Agent": "DDPG"}, {"env_step": 135000, "rew": 8300.1575, "rew_std": 2756.711, "Agent": "DDPG"}, {"env_step": 140000, "rew": 8299.4218, "rew_std": 2757.5534, "Agent": "DDPG"}, {"env_step": 145000, "rew": 8298.7566, "rew_std": 2756.7844, "Agent": "DDPG"}, {"env_step": 150000, "rew": 8207.5663, "rew_std": 2740.2049, "Agent": "DDPG"}, {"env_step": 155000, "rew": 8297.8394, "rew_std": 2755.9258, "Agent": "DDPG"}, {"env_step": 160000, "rew": 8296.2179, "rew_std": 2755.9276, "Agent": "DDPG"}, {"env_step": 165000, "rew": 8295.9536, "rew_std": 2756.1112, "Agent": "DDPG"}, {"env_step": 170000, "rew": 8295.9494, "rew_std": 2756.3831, "Agent": "DDPG"}, {"env_step": 175000, "rew": 8293.5577, "rew_std": 2755.3327, "Agent": "DDPG"}, {"env_step": 180000, "rew": 8292.3673, "rew_std": 2754.9256, "Agent": "DDPG"}, {"env_step": 185000, "rew": 8292.4829, "rew_std": 2754.6934, "Agent": "DDPG"}, {"env_step": 190000, "rew": 8292.4845, "rew_std": 2755.2319, "Agent": "DDPG"}, {"env_step": 195000, "rew": 8295.9467, "rew_std": 2755.58, "Agent": "DDPG"}, {"env_step": 200000, "rew": 8293.4849, "rew_std": 2755.315, "Agent": "DDPG"}, {"env_step": 205000, "rew": 8295.1456, "rew_std": 2755.5937, "Agent": "DDPG"}, {"env_step": 210000, "rew": 8292.4386, "rew_std": 2753.8752, "Agent": "DDPG"}, {"env_step": 215000, "rew": 8292.1814, "rew_std": 2754.5895, "Agent": "DDPG"}, {"env_step": 220000, "rew": 8294.0952, "rew_std": 2754.9637, "Agent": "DDPG"}, {"env_step": 225000, "rew": 8294.9958, "rew_std": 2754.735, "Agent": "DDPG"}, {"env_step": 230000, "rew": 8292.6771, "rew_std": 2755.0272, "Agent": "DDPG"}, {"env_step": 235000, "rew": 8290.456, "rew_std": 2754.5529, "Agent": "DDPG"}, {"env_step": 240000, "rew": 8292.1381, "rew_std": 2754.8685, "Agent": "DDPG"}, {"env_step": 245000, "rew": 8291.5933, "rew_std": 2754.1325, "Agent": "DDPG"}, {"env_step": 250000, "rew": 8292.011, "rew_std": 2754.5416, "Agent": "DDPG"}, {"env_step": 255000, "rew": 8293.3528, "rew_std": 2755.5253, "Agent": "DDPG"}, {"env_step": 260000, "rew": 8293.4076, "rew_std": 2754.9993, "Agent": "DDPG"}, {"env_step": 265000, "rew": 8290.5226, "rew_std": 2754.3122, "Agent": "DDPG"}, {"env_step": 270000, "rew": 8293.537, "rew_std": 2755.038, "Agent": "DDPG"}, {"env_step": 275000, "rew": 8289.003, "rew_std": 2753.8052, "Agent": "DDPG"}, {"env_step": 280000, "rew": 8291.2201, "rew_std": 2754.283, "Agent": "DDPG"}, {"env_step": 285000, "rew": 8291.5624, "rew_std": 2754.3976, "Agent": "DDPG"}, {"env_step": 290000, "rew": 8292.1208, "rew_std": 2754.8444, "Agent": "DDPG"}, {"env_step": 295000, "rew": 8292.2593, "rew_std": 2754.084, "Agent": "DDPG"}, {"env_step": 300000, "rew": 8292.862, "rew_std": 2754.8324, "Agent": "DDPG"}, {"env_step": 305000, "rew": 8291.4754, "rew_std": 2753.8274, "Agent": "DDPG"}, {"env_step": 310000, "rew": 8292.0324, "rew_std": 2754.5388, "Agent": "DDPG"}, {"env_step": 315000, "rew": 8291.5211, "rew_std": 2754.1126, "Agent": "DDPG"}, {"env_step": 320000, "rew": 8295.0906, "rew_std": 2755.3129, "Agent": "DDPG"}, {"env_step": 325000, "rew": 8290.7056, "rew_std": 2754.103, "Agent": "DDPG"}, {"env_step": 330000, "rew": 8289.1691, "rew_std": 2753.597, "Agent": "DDPG"}, {"env_step": 335000, "rew": 8291.5639, "rew_std": 2754.6603, "Agent": "DDPG"}, {"env_step": 340000, "rew": 8291.2692, "rew_std": 2754.5552, "Agent": "DDPG"}, {"env_step": 345000, "rew": 8291.326, "rew_std": 2754.592, "Agent": "DDPG"}, {"env_step": 350000, "rew": 8290.4233, "rew_std": 2754.2992, "Agent": "DDPG"}, {"env_step": 355000, "rew": 8288.9633, "rew_std": 2753.7956, "Agent": "DDPG"}, {"env_step": 360000, "rew": 8116.2298, "rew_std": 2746.018, "Agent": "DDPG"}, {"env_step": 365000, "rew": 8292.9889, "rew_std": 2755.1284, "Agent": "DDPG"}, {"env_step": 370000, "rew": 8292.0499, "rew_std": 2754.5696, "Agent": "DDPG"}, {"env_step": 375000, "rew": 8291.2033, "rew_std": 2754.554, "Agent": "DDPG"}, {"env_step": 380000, "rew": 8289.8346, "rew_std": 2754.3545, "Agent": "DDPG"}, {"env_step": 385000, "rew": 8291.7698, "rew_std": 2755.0038, "Agent": "DDPG"}, {"env_step": 390000, "rew": 8292.5342, "rew_std": 2755.2758, "Agent": "DDPG"}, {"env_step": 395000, "rew": 8291.0649, "rew_std": 2754.2264, "Agent": "DDPG"}, {"env_step": 400000, "rew": 8291.7335, "rew_std": 2754.4616, "Agent": "DDPG"}, {"env_step": 405000, "rew": 8292.3161, "rew_std": 2755.182, "Agent": "DDPG"}, {"env_step": 410000, "rew": 8292.905, "rew_std": 2755.1138, "Agent": "DDPG"}, {"env_step": 415000, "rew": 8292.1131, "rew_std": 2754.0447, "Agent": "DDPG"}, {"env_step": 420000, "rew": 8296.2261, "rew_std": 2755.6873, "Agent": "DDPG"}, {"env_step": 425000, "rew": 8292.7841, "rew_std": 2754.7871, "Agent": "DDPG"}, {"env_step": 430000, "rew": 8289.4164, "rew_std": 2754.2258, "Agent": "DDPG"}, {"env_step": 435000, "rew": 8290.0856, "rew_std": 2754.4358, "Agent": "DDPG"}, {"env_step": 440000, "rew": 8291.561, "rew_std": 2754.9278, "Agent": "DDPG"}, {"env_step": 445000, "rew": 8292.6104, "rew_std": 2754.7454, "Agent": "DDPG"}, {"env_step": 450000, "rew": 8293.0174, "rew_std": 2754.603, "Agent": "DDPG"}, {"env_step": 455000, "rew": 8290.9708, "rew_std": 2753.6444, "Agent": "DDPG"}, {"env_step": 460000, "rew": 8297.6342, "rew_std": 2756.4137, "Agent": "DDPG"}, {"env_step": 465000, "rew": 8291.2503, "rew_std": 2754.0156, "Agent": "DDPG"}, {"env_step": 470000, "rew": 8291.3019, "rew_std": 2754.5721, "Agent": "DDPG"}, {"env_step": 475000, "rew": 8292.475, "rew_std": 2755.2348, "Agent": "DDPG"}, {"env_step": 480000, "rew": 8294.4008, "rew_std": 2755.0725, "Agent": "DDPG"}, {"env_step": 485000, "rew": 8291.7367, "rew_std": 2754.7268, "Agent": "DDPG"}, {"env_step": 490000, "rew": 8293.2381, "rew_std": 2755.4898, "Agent": "DDPG"}, {"env_step": 495000, "rew": 8294.7888, "rew_std": 2755.4598, "Agent": "DDPG"}, {"env_step": 500000, "rew": 8296.2677, "rew_std": 2756.2544, "Agent": "DDPG"}, {"env_step": 505000, "rew": 8296.3363, "rew_std": 2756.2511, "Agent": "DDPG"}, {"env_step": 510000, "rew": 8298.5078, "rew_std": 2756.976, "Agent": "DDPG"}, {"env_step": 515000, "rew": 8290.4035, "rew_std": 2753.7381, "Agent": "DDPG"}, {"env_step": 520000, "rew": 8293.848, "rew_std": 2755.4232, "Agent": "DDPG"}, {"env_step": 525000, "rew": 8293.535, "rew_std": 2755.0527, "Agent": "DDPG"}, {"env_step": 530000, "rew": 8293.2774, "rew_std": 2754.9621, "Agent": "DDPG"}, {"env_step": 535000, "rew": 8292.2477, "rew_std": 2754.3555, "Agent": "DDPG"}, {"env_step": 540000, "rew": 8292.3039, "rew_std": 2754.3911, "Agent": "DDPG"}, {"env_step": 545000, "rew": 8292.5461, "rew_std": 2755.2645, "Agent": "DDPG"}, {"env_step": 550000, "rew": 8292.4917, "rew_std": 2754.9733, "Agent": "DDPG"}, {"env_step": 555000, "rew": 8292.9037, "rew_std": 2754.8334, "Agent": "DDPG"}, {"env_step": 560000, "rew": 8294.958, "rew_std": 2755.7869, "Agent": "DDPG"}, {"env_step": 565000, "rew": 8293.528, "rew_std": 2755.0432, "Agent": "DDPG"}, {"env_step": 570000, "rew": 8299.4294, "rew_std": 2757.0191, "Agent": "DDPG"}, {"env_step": 575000, "rew": 8293.6638, "rew_std": 2755.0938, "Agent": "DDPG"}, {"env_step": 580000, "rew": 8291.0625, "rew_std": 2754.5043, "Agent": "DDPG"}, {"env_step": 585000, "rew": 8294.6894, "rew_std": 2755.9684, "Agent": "DDPG"}, {"env_step": 590000, "rew": 8295.7791, "rew_std": 2755.8167, "Agent": "DDPG"}, {"env_step": 595000, "rew": 8292.8612, "rew_std": 2755.1106, "Agent": "DDPG"}, {"env_step": 600000, "rew": 8291.6771, "rew_std": 2753.8879, "Agent": "DDPG"}, {"env_step": 605000, "rew": 8295.3033, "rew_std": 2755.9122, "Agent": "DDPG"}, {"env_step": 610000, "rew": 8296.0122, "rew_std": 2756.4109, "Agent": "DDPG"}, {"env_step": 615000, "rew": 8295.3513, "rew_std": 2755.9098, "Agent": "DDPG"}, {"env_step": 620000, "rew": 8294.0031, "rew_std": 2754.9571, "Agent": "DDPG"}, {"env_step": 625000, "rew": 8292.0084, "rew_std": 2754.2688, "Agent": "DDPG"}, {"env_step": 630000, "rew": 8293.83, "rew_std": 2755.1403, "Agent": "DDPG"}, {"env_step": 635000, "rew": 8295.4863, "rew_std": 2755.9707, "Agent": "DDPG"}, {"env_step": 640000, "rew": 8293.4934, "rew_std": 2755.3081, "Agent": "DDPG"}, {"env_step": 645000, "rew": 8295.3632, "rew_std": 2755.9298, "Agent": "DDPG"}, {"env_step": 650000, "rew": 8296.3766, "rew_std": 2756.2698, "Agent": "DDPG"}, {"env_step": 655000, "rew": 8291.4066, "rew_std": 2754.0746, "Agent": "DDPG"}, {"env_step": 660000, "rew": 8293.2079, "rew_std": 2754.6713, "Agent": "DDPG"}, {"env_step": 665000, "rew": 8293.3277, "rew_std": 2754.9914, "Agent": "DDPG"}, {"env_step": 670000, "rew": 8292.8327, "rew_std": 2754.8237, "Agent": "DDPG"}, {"env_step": 675000, "rew": 8293.4509, "rew_std": 2755.3201, "Agent": "DDPG"}, {"env_step": 680000, "rew": 8297.0175, "rew_std": 2756.2111, "Agent": "DDPG"}, {"env_step": 685000, "rew": 8294.7786, "rew_std": 2755.2036, "Agent": "DDPG"}, {"env_step": 690000, "rew": 8295.8477, "rew_std": 2756.0947, "Agent": "DDPG"}, {"env_step": 695000, "rew": 8297.5199, "rew_std": 2756.1001, "Agent": "DDPG"}, {"env_step": 700000, "rew": 8294.6571, "rew_std": 2755.1663, "Agent": "DDPG"}, {"env_step": 705000, "rew": 8293.7032, "rew_std": 2755.1142, "Agent": "DDPG"}, {"env_step": 710000, "rew": 8297.6848, "rew_std": 2756.4248, "Agent": "DDPG"}, {"env_step": 715000, "rew": 8296.3341, "rew_std": 2756.2396, "Agent": "DDPG"}, {"env_step": 720000, "rew": 8297.6563, "rew_std": 2756.4191, "Agent": "DDPG"}, {"env_step": 725000, "rew": 8295.9062, "rew_std": 2756.3863, "Agent": "DDPG"}, {"env_step": 730000, "rew": 8295.1217, "rew_std": 2755.5904, "Agent": "DDPG"}, {"env_step": 735000, "rew": 8295.437, "rew_std": 2756.2321, "Agent": "DDPG"}, {"env_step": 740000, "rew": 8297.9849, "rew_std": 2756.8038, "Agent": "DDPG"}, {"env_step": 745000, "rew": 8295.0699, "rew_std": 2756.0934, "Agent": "DDPG"}, {"env_step": 750000, "rew": 8295.2069, "rew_std": 2755.054, "Agent": "DDPG"}, {"env_step": 755000, "rew": 8294.4887, "rew_std": 2755.6316, "Agent": "DDPG"}, {"env_step": 760000, "rew": 8295.5198, "rew_std": 2756.25, "Agent": "DDPG"}, {"env_step": 765000, "rew": 8298.3552, "rew_std": 2756.9195, "Agent": "DDPG"}, {"env_step": 770000, "rew": 8296.7929, "rew_std": 2756.1303, "Agent": "DDPG"}, {"env_step": 775000, "rew": 8299.761, "rew_std": 2756.8651, "Agent": "DDPG"}, {"env_step": 780000, "rew": 8298.0048, "rew_std": 2756.8088, "Agent": "DDPG"}, {"env_step": 785000, "rew": 8296.2551, "rew_std": 2755.9583, "Agent": "DDPG"}, {"env_step": 790000, "rew": 8295.302, "rew_std": 2755.6306, "Agent": "DDPG"}, {"env_step": 795000, "rew": 8292.0512, "rew_std": 2754.5373, "Agent": "DDPG"}, {"env_step": 800000, "rew": 8294.4388, "rew_std": 2755.6203, "Agent": "DDPG"}, {"env_step": 805000, "rew": 8297.1481, "rew_std": 2756.5269, "Agent": "DDPG"}, {"env_step": 810000, "rew": 8292.8083, "rew_std": 2755.3493, "Agent": "DDPG"}, {"env_step": 815000, "rew": 8301.0287, "rew_std": 2757.8421, "Agent": "DDPG"}, {"env_step": 820000, "rew": 8295.6358, "rew_std": 2755.7482, "Agent": "DDPG"}, {"env_step": 825000, "rew": 8291.6853, "rew_std": 2754.6938, "Agent": "DDPG"}, {"env_step": 830000, "rew": 8295.6276, "rew_std": 2755.7279, "Agent": "DDPG"}, {"env_step": 835000, "rew": 8295.8508, "rew_std": 2756.3616, "Agent": "DDPG"}, {"env_step": 840000, "rew": 8296.7933, "rew_std": 2756.6663, "Agent": "DDPG"}, {"env_step": 845000, "rew": 8297.2022, "rew_std": 2756.7935, "Agent": "DDPG"}, {"env_step": 850000, "rew": 8296.1111, "rew_std": 2756.1792, "Agent": "DDPG"}, {"env_step": 855000, "rew": 8296.2931, "rew_std": 2756.5108, "Agent": "DDPG"}, {"env_step": 860000, "rew": 8295.5999, "rew_std": 2756.2757, "Agent": "DDPG"}, {"env_step": 865000, "rew": 8300.1544, "rew_std": 2757.5203, "Agent": "DDPG"}, {"env_step": 870000, "rew": 8296.1314, "rew_std": 2755.9105, "Agent": "DDPG"}, {"env_step": 875000, "rew": 8297.2111, "rew_std": 2756.5472, "Agent": "DDPG"}, {"env_step": 880000, "rew": 8295.3819, "rew_std": 2754.5896, "Agent": "DDPG"}, {"env_step": 885000, "rew": 8296.8615, "rew_std": 2756.429, "Agent": "DDPG"}, {"env_step": 890000, "rew": 8298.6941, "rew_std": 2756.757, "Agent": "DDPG"}, {"env_step": 895000, "rew": 8297.697, "rew_std": 2756.437, "Agent": "DDPG"}, {"env_step": 900000, "rew": 8296.1023, "rew_std": 2755.8934, "Agent": "DDPG"}, {"env_step": 905000, "rew": 8298.3042, "rew_std": 2756.369, "Agent": "DDPG"}, {"env_step": 910000, "rew": 8296.4108, "rew_std": 2756.0118, "Agent": "DDPG"}, {"env_step": 915000, "rew": 8298.7159, "rew_std": 2757.064, "Agent": "DDPG"}, {"env_step": 920000, "rew": 8298.8204, "rew_std": 2757.3504, "Agent": "DDPG"}, {"env_step": 925000, "rew": 8300.3121, "rew_std": 2757.3023, "Agent": "DDPG"}, {"env_step": 930000, "rew": 8296.4645, "rew_std": 2756.0223, "Agent": "DDPG"}, {"env_step": 935000, "rew": 8300.1583, "rew_std": 2757.5244, "Agent": "DDPG"}, {"env_step": 940000, "rew": 8294.9937, "rew_std": 2755.7992, "Agent": "DDPG"}, {"env_step": 945000, "rew": 8297.501, "rew_std": 2756.9001, "Agent": "DDPG"}, {"env_step": 950000, "rew": 8295.0375, "rew_std": 2755.5536, "Agent": "DDPG"}, {"env_step": 955000, "rew": 8295.7071, "rew_std": 2755.5132, "Agent": "DDPG"}, {"env_step": 960000, "rew": 8302.3607, "rew_std": 2758.2533, "Agent": "DDPG"}, {"env_step": 965000, "rew": 8296.644, "rew_std": 2756.3489, "Agent": "DDPG"}, {"env_step": 970000, "rew": 8298.3855, "rew_std": 2756.926, "Agent": "DDPG"}, {"env_step": 975000, "rew": 8297.1022, "rew_std": 2756.7851, "Agent": "DDPG"}, {"env_step": 980000, "rew": 8300.2825, "rew_std": 2757.8324, "Agent": "DDPG"}, {"env_step": 985000, "rew": 8297.9532, "rew_std": 2756.5307, "Agent": "DDPG"}, {"env_step": 990000, "rew": 8296.4771, "rew_std": 2756.0346, "Agent": "DDPG"}, {"env_step": 995000, "rew": 8295.8242, "rew_std": 2756.0726, "Agent": "DDPG"}, {"env_step": 1000000, "rew": 8296.892, "rew_std": 2756.4241, "Agent": "DDPG"}, {"env_step": 1005000, "rew": 8297.6683, "rew_std": 2756.1623, "Agent": "DDPG"}, {"env_step": 1010000, "rew": 8295.0875, "rew_std": 2755.8313, "Agent": "DDPG"}, {"env_step": 1015000, "rew": 8297.3172, "rew_std": 2756.5848, "Agent": "DDPG"}, {"env_step": 1020000, "rew": 8293.767, "rew_std": 2755.3931, "Agent": "DDPG"}, {"env_step": 1025000, "rew": 8293.8516, "rew_std": 2755.699, "Agent": "DDPG"}, {"env_step": 1030000, "rew": 8292.2973, "rew_std": 2754.3564, "Agent": "DDPG"}, {"env_step": 1035000, "rew": 8293.4526, "rew_std": 2755.2837, "Agent": "DDPG"}, {"env_step": 1040000, "rew": 8293.6104, "rew_std": 2755.3343, "Agent": "DDPG"}, {"env_step": 1045000, "rew": 8293.7552, "rew_std": 2754.8439, "Agent": "DDPG"}, {"env_step": 1050000, "rew": 7662.5222, "rew_std": 3164.0049, "Agent": "DDPG"}, {"env_step": 1055000, "rew": 8294.0077, "rew_std": 2754.9234, "Agent": "DDPG"}, {"env_step": 1060000, "rew": 8293.2793, "rew_std": 2755.2315, "Agent": "DDPG"}, {"env_step": 1065000, "rew": 8292.5417, "rew_std": 2755.2513, "Agent": "DDPG"}, {"env_step": 1070000, "rew": 8292.0735, "rew_std": 2754.2833, "Agent": "DDPG"}, {"env_step": 1075000, "rew": 8293.0015, "rew_std": 2754.6026, "Agent": "DDPG"}, {"env_step": 1080000, "rew": 8291.2618, "rew_std": 2754.5517, "Agent": "DDPG"}, {"env_step": 1085000, "rew": 8292.1272, "rew_std": 2754.8365, "Agent": "DDPG"}, {"env_step": 1090000, "rew": 8291.3055, "rew_std": 2754.0306, "Agent": "DDPG"}, {"env_step": 1095000, "rew": 8292.1366, "rew_std": 2754.5853, "Agent": "DDPG"}, {"env_step": 1100000, "rew": 8292.0401, "rew_std": 2755.0818, "Agent": "DDPG"}, {"env_step": 1105000, "rew": 8291.6328, "rew_std": 2754.9401, "Agent": "DDPG"}, {"env_step": 1110000, "rew": 8290.2115, "rew_std": 2753.9331, "Agent": "DDPG"}, {"env_step": 1115000, "rew": 8290.8631, "rew_std": 2754.4196, "Agent": "DDPG"}, {"env_step": 1120000, "rew": 8289.9777, "rew_std": 2753.5913, "Agent": "DDPG"}, {"env_step": 1125000, "rew": 8292.2272, "rew_std": 2754.345, "Agent": "DDPG"}, {"env_step": 1130000, "rew": 8290.1447, "rew_std": 2754.1867, "Agent": "DDPG"}, {"env_step": 1135000, "rew": 8290.2864, "rew_std": 2753.9518, "Agent": "DDPG"}, {"env_step": 1140000, "rew": 8289.5183, "rew_std": 2753.9746, "Agent": "DDPG"}, {"env_step": 1145000, "rew": 8288.7194, "rew_std": 2753.4345, "Agent": "DDPG"}, {"env_step": 1150000, "rew": 8291.2075, "rew_std": 2754.2573, "Agent": "DDPG"}, {"env_step": 1155000, "rew": 8290.7335, "rew_std": 2753.8476, "Agent": "DDPG"}, {"env_step": 1160000, "rew": 8293.1154, "rew_std": 2754.9056, "Agent": "DDPG"}, {"env_step": 1165000, "rew": 8290.8389, "rew_std": 2753.8674, "Agent": "DDPG"}, {"env_step": 1170000, "rew": 8290.7747, "rew_std": 2754.1264, "Agent": "DDPG"}, {"env_step": 1175000, "rew": 8254.041, "rew_std": 2744.3561, "Agent": "DDPG"}, {"env_step": 1180000, "rew": 8292.2006, "rew_std": 2754.6063, "Agent": "DDPG"}, {"env_step": 1185000, "rew": 8290.7599, "rew_std": 2754.396, "Agent": "DDPG"}, {"env_step": 1190000, "rew": 8290.1479, "rew_std": 2753.3733, "Agent": "DDPG"}, {"env_step": 1195000, "rew": 8291.7456, "rew_std": 2754.7311, "Agent": "DDPG"}, {"env_step": 1200000, "rew": 8293.014, "rew_std": 2754.6125, "Agent": "DDPG"}, {"env_step": 1205000, "rew": 8290.6384, "rew_std": 2754.3514, "Agent": "DDPG"}, {"env_step": 1210000, "rew": 8291.264, "rew_std": 2754.8298, "Agent": "DDPG"}, {"env_step": 1215000, "rew": 8289.5644, "rew_std": 2753.9916, "Agent": "DDPG"}, {"env_step": 1220000, "rew": 8292.3213, "rew_std": 2755.2023, "Agent": "DDPG"}, {"env_step": 1225000, "rew": 8292.2093, "rew_std": 2755.1442, "Agent": "DDPG"}, {"env_step": 1230000, "rew": 8292.6926, "rew_std": 2754.7738, "Agent": "DDPG"}, {"env_step": 1235000, "rew": 8289.2696, "rew_std": 2754.1698, "Agent": "DDPG"}, {"env_step": 1240000, "rew": 8293.3196, "rew_std": 2754.9889, "Agent": "DDPG"}, {"env_step": 1245000, "rew": 8291.3114, "rew_std": 2754.3019, "Agent": "DDPG"}, {"env_step": 1250000, "rew": 8294.6964, "rew_std": 2755.7174, "Agent": "DDPG"}, {"env_step": 5000, "rew": 282.9489, "rew_std": 105.3973, "Agent": "TD3"}, {"env_step": 10000, "rew": 967.198, "rew_std": 587.396, "Agent": "TD3"}, {"env_step": 15000, "rew": 2603.2403, "rew_std": 2591.978, "Agent": "TD3"}, {"env_step": 20000, "rew": 7167.7542, "rew_std": 3486.9054, "Agent": "TD3"}, {"env_step": 25000, "rew": 8426.4864, "rew_std": 2799.8999, "Agent": "TD3"}, {"env_step": 30000, "rew": 8421.4054, "rew_std": 2798.2367, "Agent": "TD3"}, {"env_step": 35000, "rew": 8434.9704, "rew_std": 2749.6202, "Agent": "TD3"}, {"env_step": 40000, "rew": 8462.4527, "rew_std": 2679.5802, "Agent": "TD3"}, {"env_step": 45000, "rew": 9264.4194, "rew_std": 264.6839, "Agent": "TD3"}, {"env_step": 50000, "rew": 9342.7878, "rew_std": 16.0342, "Agent": "TD3"}, {"env_step": 55000, "rew": 9349.233, "rew_std": 14.2728, "Agent": "TD3"}, {"env_step": 60000, "rew": 9251.8021, "rew_std": 277.0279, "Agent": "TD3"}, {"env_step": 65000, "rew": 9063.1921, "rew_std": 408.0763, "Agent": "TD3"}, {"env_step": 70000, "rew": 9239.0729, "rew_std": 268.5189, "Agent": "TD3"}, {"env_step": 75000, "rew": 9315.5051, "rew_std": 10.5088, "Agent": "TD3"}, {"env_step": 80000, "rew": 9311.2797, "rew_std": 3.2938, "Agent": "TD3"}, {"env_step": 85000, "rew": 9315.4548, "rew_std": 14.4811, "Agent": "TD3"}, {"env_step": 90000, "rew": 9315.1414, "rew_std": 14.8833, "Agent": "TD3"}, {"env_step": 95000, "rew": 9316.2116, "rew_std": 14.8043, "Agent": "TD3"}, {"env_step": 100000, "rew": 9315.5365, "rew_std": 14.7679, "Agent": "TD3"}, {"env_step": 105000, "rew": 9315.3719, "rew_std": 14.7958, "Agent": "TD3"}, {"env_step": 110000, "rew": 9314.6993, "rew_std": 14.8525, "Agent": "TD3"}, {"env_step": 115000, "rew": 9315.4553, "rew_std": 14.81, "Agent": "TD3"}, {"env_step": 120000, "rew": 9314.5234, "rew_std": 14.8132, "Agent": "TD3"}, {"env_step": 125000, "rew": 9313.8028, "rew_std": 12.1399, "Agent": "TD3"}, {"env_step": 130000, "rew": 9314.5627, "rew_std": 14.6963, "Agent": "TD3"}, {"env_step": 135000, "rew": 9312.9483, "rew_std": 9.0585, "Agent": "TD3"}, {"env_step": 140000, "rew": 9229.2644, "rew_std": 241.3705, "Agent": "TD3"}, {"env_step": 145000, "rew": 9309.6136, "rew_std": 0.7793, "Agent": "TD3"}, {"env_step": 150000, "rew": 9310.1716, "rew_std": 1.6606, "Agent": "TD3"}, {"env_step": 155000, "rew": 9311.2081, "rew_std": 1.3916, "Agent": "TD3"}, {"env_step": 160000, "rew": 8471.2184, "rew_std": 2516.0911, "Agent": "TD3"}, {"env_step": 165000, "rew": 8036.5291, "rew_std": 2707.4981, "Agent": "TD3"}, {"env_step": 170000, "rew": 8293.0254, "rew_std": 2553.5993, "Agent": "TD3"}, {"env_step": 175000, "rew": 8314.9329, "rew_std": 2476.8869, "Agent": "TD3"}, {"env_step": 180000, "rew": 8455.8738, "rew_std": 2564.073, "Agent": "TD3"}, {"env_step": 185000, "rew": 8452.5523, "rew_std": 2571.196, "Agent": "TD3"}, {"env_step": 190000, "rew": 8467.5006, "rew_std": 2527.4494, "Agent": "TD3"}, {"env_step": 195000, "rew": 9309.6068, "rew_std": 0.6428, "Agent": "TD3"}, {"env_step": 200000, "rew": 9309.982, "rew_std": 1.4155, "Agent": "TD3"}, {"env_step": 205000, "rew": 9310.3987, "rew_std": 2.7655, "Agent": "TD3"}, {"env_step": 210000, "rew": 9310.9153, "rew_std": 2.8681, "Agent": "TD3"}, {"env_step": 215000, "rew": 9310.6003, "rew_std": 2.9948, "Agent": "TD3"}, {"env_step": 220000, "rew": 9309.9613, "rew_std": 1.6709, "Agent": "TD3"}, {"env_step": 225000, "rew": 9310.7892, "rew_std": 2.3935, "Agent": "TD3"}, {"env_step": 230000, "rew": 9310.0957, "rew_std": 1.6486, "Agent": "TD3"}, {"env_step": 235000, "rew": 9313.6597, "rew_std": 10.337, "Agent": "TD3"}, {"env_step": 240000, "rew": 9309.9069, "rew_std": 2.888, "Agent": "TD3"}, {"env_step": 245000, "rew": 9223.8129, "rew_std": 257.1406, "Agent": "TD3"}, {"env_step": 250000, "rew": 9310.8441, "rew_std": 2.4554, "Agent": "TD3"}, {"env_step": 255000, "rew": 9310.2397, "rew_std": 2.7111, "Agent": "TD3"}, {"env_step": 260000, "rew": 9310.0834, "rew_std": 1.0443, "Agent": "TD3"}, {"env_step": 265000, "rew": 9310.5597, "rew_std": 2.2972, "Agent": "TD3"}, {"env_step": 270000, "rew": 9310.237, "rew_std": 3.2016, "Agent": "TD3"}, {"env_step": 275000, "rew": 9310.3845, "rew_std": 2.2792, "Agent": "TD3"}, {"env_step": 280000, "rew": 9312.2651, "rew_std": 6.7065, "Agent": "TD3"}, {"env_step": 285000, "rew": 9312.2881, "rew_std": 6.6489, "Agent": "TD3"}, {"env_step": 290000, "rew": 9310.0907, "rew_std": 3.6352, "Agent": "TD3"}, {"env_step": 295000, "rew": 9314.1539, "rew_std": 15.0218, "Agent": "TD3"}, {"env_step": 300000, "rew": 9051.2807, "rew_std": 775.6418, "Agent": "TD3"}, {"env_step": 305000, "rew": 8184.3012, "rew_std": 2622.3709, "Agent": "TD3"}, {"env_step": 310000, "rew": 8874.6879, "rew_std": 1312.1528, "Agent": "TD3"}, {"env_step": 315000, "rew": 9314.4322, "rew_std": 14.1116, "Agent": "TD3"}, {"env_step": 320000, "rew": 9312.8483, "rew_std": 7.482, "Agent": "TD3"}, {"env_step": 325000, "rew": 9313.6615, "rew_std": 8.8537, "Agent": "TD3"}, {"env_step": 330000, "rew": 9312.3362, "rew_std": 9.6878, "Agent": "TD3"}, {"env_step": 335000, "rew": 9311.1867, "rew_std": 4.9263, "Agent": "TD3"}, {"env_step": 340000, "rew": 9311.1632, "rew_std": 4.0272, "Agent": "TD3"}, {"env_step": 345000, "rew": 9311.8437, "rew_std": 3.7329, "Agent": "TD3"}, {"env_step": 350000, "rew": 8525.1915, "rew_std": 2368.4822, "Agent": "TD3"}, {"env_step": 355000, "rew": 9314.4612, "rew_std": 11.0448, "Agent": "TD3"}, {"env_step": 360000, "rew": 9311.3431, "rew_std": 6.6012, "Agent": "TD3"}, {"env_step": 365000, "rew": 9311.3624, "rew_std": 5.7543, "Agent": "TD3"}, {"env_step": 370000, "rew": 8267.0577, "rew_std": 2620.0102, "Agent": "TD3"}, {"env_step": 375000, "rew": 8436.7321, "rew_std": 2631.9562, "Agent": "TD3"}, {"env_step": 380000, "rew": 9144.4516, "rew_std": 514.1179, "Agent": "TD3"}, {"env_step": 385000, "rew": 9313.9002, "rew_std": 14.1223, "Agent": "TD3"}, {"env_step": 390000, "rew": 9145.3532, "rew_std": 507.4234, "Agent": "TD3"}, {"env_step": 395000, "rew": 8885.2745, "rew_std": 1022.9433, "Agent": "TD3"}, {"env_step": 400000, "rew": 9099.8592, "rew_std": 633.7039, "Agent": "TD3"}, {"env_step": 405000, "rew": 9060.6217, "rew_std": 769.8305, "Agent": "TD3"}, {"env_step": 410000, "rew": 9317.8565, "rew_std": 15.6384, "Agent": "TD3"}, {"env_step": 415000, "rew": 9317.4512, "rew_std": 15.2292, "Agent": "TD3"}, {"env_step": 420000, "rew": 9315.6176, "rew_std": 13.6355, "Agent": "TD3"}, {"env_step": 425000, "rew": 8472.1398, "rew_std": 2542.5459, "Agent": "TD3"}, {"env_step": 430000, "rew": 8639.4106, "rew_std": 2031.7747, "Agent": "TD3"}, {"env_step": 435000, "rew": 9319.8521, "rew_std": 18.4469, "Agent": "TD3"}, {"env_step": 440000, "rew": 9316.8172, "rew_std": 16.1172, "Agent": "TD3"}, {"env_step": 445000, "rew": 9319.7009, "rew_std": 20.0731, "Agent": "TD3"}, {"env_step": 450000, "rew": 9066.934, "rew_std": 762.7563, "Agent": "TD3"}, {"env_step": 455000, "rew": 8814.4498, "rew_std": 1510.9292, "Agent": "TD3"}, {"env_step": 460000, "rew": 9320.5916, "rew_std": 18.5887, "Agent": "TD3"}, {"env_step": 465000, "rew": 9318.9374, "rew_std": 18.6527, "Agent": "TD3"}, {"env_step": 470000, "rew": 8383.681, "rew_std": 2528.0881, "Agent": "TD3"}, {"env_step": 475000, "rew": 9320.2665, "rew_std": 16.8646, "Agent": "TD3"}, {"env_step": 480000, "rew": 9320.4374, "rew_std": 17.5623, "Agent": "TD3"}, {"env_step": 485000, "rew": 8444.0166, "rew_std": 2625.2926, "Agent": "TD3"}, {"env_step": 490000, "rew": 9313.9254, "rew_std": 12.639, "Agent": "TD3"}, {"env_step": 495000, "rew": 9152.4438, "rew_std": 507.472, "Agent": "TD3"}, {"env_step": 500000, "rew": 9320.859, "rew_std": 12.8297, "Agent": "TD3"}, {"env_step": 505000, "rew": 9238.7633, "rew_std": 252.8251, "Agent": "TD3"}, {"env_step": 510000, "rew": 8614.0148, "rew_std": 2124.8329, "Agent": "TD3"}, {"env_step": 515000, "rew": 9322.0179, "rew_std": 17.8638, "Agent": "TD3"}, {"env_step": 520000, "rew": 9323.2759, "rew_std": 19.0967, "Agent": "TD3"}, {"env_step": 525000, "rew": 9328.2951, "rew_std": 20.2475, "Agent": "TD3"}, {"env_step": 530000, "rew": 7765.9104, "rew_std": 3149.1228, "Agent": "TD3"}, {"env_step": 535000, "rew": 9327.4452, "rew_std": 20.3214, "Agent": "TD3"}, {"env_step": 540000, "rew": 9329.074, "rew_std": 20.3596, "Agent": "TD3"}, {"env_step": 545000, "rew": 9325.3436, "rew_std": 17.1185, "Agent": "TD3"}, {"env_step": 550000, "rew": 8989.8726, "rew_std": 1031.1772, "Agent": "TD3"}, {"env_step": 555000, "rew": 9331.0296, "rew_std": 21.2146, "Agent": "TD3"}, {"env_step": 560000, "rew": 9330.7565, "rew_std": 19.3693, "Agent": "TD3"}, {"env_step": 565000, "rew": 9329.7398, "rew_std": 20.3798, "Agent": "TD3"}, {"env_step": 570000, "rew": 9325.1319, "rew_std": 19.1622, "Agent": "TD3"}, {"env_step": 575000, "rew": 8480.9527, "rew_std": 2520.0155, "Agent": "TD3"}, {"env_step": 580000, "rew": 9321.9878, "rew_std": 15.8911, "Agent": "TD3"}, {"env_step": 585000, "rew": 8391.7263, "rew_std": 2485.2671, "Agent": "TD3"}, {"env_step": 590000, "rew": 9320.116, "rew_std": 13.4094, "Agent": "TD3"}, {"env_step": 595000, "rew": 8479.5073, "rew_std": 2522.3627, "Agent": "TD3"}, {"env_step": 600000, "rew": 9321.8606, "rew_std": 14.7027, "Agent": "TD3"}, {"env_step": 605000, "rew": 9062.4936, "rew_std": 789.4461, "Agent": "TD3"}, {"env_step": 610000, "rew": 8267.8905, "rew_std": 2266.138, "Agent": "TD3"}, {"env_step": 615000, "rew": 8488.8207, "rew_std": 2511.8177, "Agent": "TD3"}, {"env_step": 620000, "rew": 9325.2056, "rew_std": 17.2205, "Agent": "TD3"}, {"env_step": 625000, "rew": 8471.006, "rew_std": 2551.4845, "Agent": "TD3"}, {"env_step": 630000, "rew": 9243.0092, "rew_std": 243.8519, "Agent": "TD3"}, {"env_step": 635000, "rew": 9323.9628, "rew_std": 18.2174, "Agent": "TD3"}, {"env_step": 640000, "rew": 9174.3501, "rew_std": 461.7308, "Agent": "TD3"}, {"env_step": 645000, "rew": 9327.2489, "rew_std": 17.2679, "Agent": "TD3"}, {"env_step": 650000, "rew": 8608.7013, "rew_std": 2165.4249, "Agent": "TD3"}, {"env_step": 655000, "rew": 8917.4672, "rew_std": 1245.5319, "Agent": "TD3"}, {"env_step": 660000, "rew": 9333.3707, "rew_std": 18.4717, "Agent": "TD3"}, {"env_step": 665000, "rew": 8293.4616, "rew_std": 2412.2377, "Agent": "TD3"}, {"env_step": 670000, "rew": 9333.9283, "rew_std": 18.5759, "Agent": "TD3"}, {"env_step": 675000, "rew": 8487.5841, "rew_std": 2523.9032, "Agent": "TD3"}, {"env_step": 680000, "rew": 8483.1521, "rew_std": 2541.145, "Agent": "TD3"}, {"env_step": 685000, "rew": 9267.9299, "rew_std": 185.369, "Agent": "TD3"}, {"env_step": 690000, "rew": 8483.5027, "rew_std": 2533.8052, "Agent": "TD3"}, {"env_step": 695000, "rew": 8395.0935, "rew_std": 2524.5447, "Agent": "TD3"}, {"env_step": 700000, "rew": 8470.1905, "rew_std": 2562.094, "Agent": "TD3"}, {"env_step": 705000, "rew": 8476.3315, "rew_std": 2549.8972, "Agent": "TD3"}, {"env_step": 710000, "rew": 8464.8866, "rew_std": 2613.7263, "Agent": "TD3"}, {"env_step": 715000, "rew": 8537.5775, "rew_std": 2385.8565, "Agent": "TD3"}, {"env_step": 720000, "rew": 8489.6683, "rew_std": 2513.7786, "Agent": "TD3"}, {"env_step": 725000, "rew": 8475.4442, "rew_std": 2568.3238, "Agent": "TD3"}, {"env_step": 730000, "rew": 8476.2893, "rew_std": 2576.5732, "Agent": "TD3"}, {"env_step": 735000, "rew": 8451.9808, "rew_std": 2637.4094, "Agent": "TD3"}, {"env_step": 740000, "rew": 8438.7148, "rew_std": 2646.9911, "Agent": "TD3"}, {"env_step": 745000, "rew": 8551.6528, "rew_std": 2306.6298, "Agent": "TD3"}, {"env_step": 750000, "rew": 9320.1892, "rew_std": 18.8426, "Agent": "TD3"}, {"env_step": 755000, "rew": 9324.6405, "rew_std": 18.1221, "Agent": "TD3"}, {"env_step": 760000, "rew": 8951.4759, "rew_std": 1113.7195, "Agent": "TD3"}, {"env_step": 765000, "rew": 9008.1179, "rew_std": 529.2366, "Agent": "TD3"}, {"env_step": 770000, "rew": 8767.8096, "rew_std": 1685.5353, "Agent": "TD3"}, {"env_step": 775000, "rew": 8495.5212, "rew_std": 2507.0633, "Agent": "TD3"}, {"env_step": 780000, "rew": 8246.6138, "rew_std": 2562.1309, "Agent": "TD3"}, {"env_step": 785000, "rew": 8475.1777, "rew_std": 2579.23, "Agent": "TD3"}, {"env_step": 790000, "rew": 8479.8606, "rew_std": 2072.6119, "Agent": "TD3"}, {"env_step": 795000, "rew": 7802.9669, "rew_std": 3063.6171, "Agent": "TD3"}, {"env_step": 800000, "rew": 8453.7153, "rew_std": 2637.2187, "Agent": "TD3"}, {"env_step": 805000, "rew": 9238.6734, "rew_std": 276.5512, "Agent": "TD3"}, {"env_step": 810000, "rew": 9333.529, "rew_std": 21.4153, "Agent": "TD3"}, {"env_step": 815000, "rew": 9327.587, "rew_std": 19.7504, "Agent": "TD3"}, {"env_step": 820000, "rew": 9330.4714, "rew_std": 20.4379, "Agent": "TD3"}, {"env_step": 825000, "rew": 9336.9744, "rew_std": 19.5782, "Agent": "TD3"}, {"env_step": 830000, "rew": 9329.7092, "rew_std": 21.4864, "Agent": "TD3"}, {"env_step": 835000, "rew": 9331.23, "rew_std": 18.5596, "Agent": "TD3"}, {"env_step": 840000, "rew": 9328.854, "rew_std": 21.1961, "Agent": "TD3"}, {"env_step": 845000, "rew": 9327.2057, "rew_std": 22.8246, "Agent": "TD3"}, {"env_step": 850000, "rew": 8903.428, "rew_std": 1283.9923, "Agent": "TD3"}, {"env_step": 855000, "rew": 8621.8355, "rew_std": 2140.9923, "Agent": "TD3"}, {"env_step": 860000, "rew": 9331.3873, "rew_std": 21.2049, "Agent": "TD3"}, {"env_step": 865000, "rew": 8512.1406, "rew_std": 2475.3905, "Agent": "TD3"}, {"env_step": 870000, "rew": 9267.4105, "rew_std": 213.968, "Agent": "TD3"}, {"env_step": 875000, "rew": 9333.9985, "rew_std": 19.9086, "Agent": "TD3"}, {"env_step": 880000, "rew": 9335.3527, "rew_std": 18.5561, "Agent": "TD3"}, {"env_step": 885000, "rew": 9162.3701, "rew_std": 536.2045, "Agent": "TD3"}, {"env_step": 890000, "rew": 8418.461, "rew_std": 2207.2014, "Agent": "TD3"}, {"env_step": 895000, "rew": 8492.1865, "rew_std": 1591.2265, "Agent": "TD3"}, {"env_step": 900000, "rew": 8446.7387, "rew_std": 2652.4248, "Agent": "TD3"}, {"env_step": 905000, "rew": 8747.1687, "rew_std": 1331.51, "Agent": "TD3"}, {"env_step": 910000, "rew": 8839.1724, "rew_std": 1487.8274, "Agent": "TD3"}, {"env_step": 915000, "rew": 9336.9595, "rew_std": 20.1037, "Agent": "TD3"}, {"env_step": 920000, "rew": 9338.5807, "rew_std": 19.6659, "Agent": "TD3"}, {"env_step": 925000, "rew": 8468.7912, "rew_std": 2574.2131, "Agent": "TD3"}, {"env_step": 930000, "rew": 9327.7642, "rew_std": 15.1276, "Agent": "TD3"}, {"env_step": 935000, "rew": 9334.2831, "rew_std": 16.6168, "Agent": "TD3"}, {"env_step": 940000, "rew": 9328.7489, "rew_std": 19.5544, "Agent": "TD3"}, {"env_step": 945000, "rew": 9333.8926, "rew_std": 21.3318, "Agent": "TD3"}, {"env_step": 950000, "rew": 8653.8862, "rew_std": 2015.9693, "Agent": "TD3"}, {"env_step": 955000, "rew": 7621.0775, "rew_std": 2831.5293, "Agent": "TD3"}, {"env_step": 960000, "rew": 9333.3804, "rew_std": 19.1393, "Agent": "TD3"}, {"env_step": 965000, "rew": 9327.7606, "rew_std": 15.0447, "Agent": "TD3"}, {"env_step": 970000, "rew": 9330.358, "rew_std": 18.4475, "Agent": "TD3"}, {"env_step": 975000, "rew": 8709.456, "rew_std": 1851.1411, "Agent": "TD3"}, {"env_step": 980000, "rew": 9333.1847, "rew_std": 19.0763, "Agent": "TD3"}, {"env_step": 985000, "rew": 9328.7851, "rew_std": 19.2249, "Agent": "TD3"}, {"env_step": 990000, "rew": 9334.8215, "rew_std": 17.4106, "Agent": "TD3"}, {"env_step": 995000, "rew": 9330.7308, "rew_std": 16.6466, "Agent": "TD3"}, {"env_step": 1000000, "rew": 9333.4001, "rew_std": 18.7685, "Agent": "TD3"}, {"env_step": 1005000, "rew": 9327.9781, "rew_std": 15.018, "Agent": "TD3"}, {"env_step": 1010000, "rew": 8995.3494, "rew_std": 764.8608, "Agent": "TD3"}, {"env_step": 1015000, "rew": 9330.6447, "rew_std": 20.8002, "Agent": "TD3"}, {"env_step": 1020000, "rew": 9323.7909, "rew_std": 15.9208, "Agent": "TD3"}, {"env_step": 1025000, "rew": 9328.5144, "rew_std": 14.8415, "Agent": "TD3"}, {"env_step": 1030000, "rew": 9329.8467, "rew_std": 17.4232, "Agent": "TD3"}, {"env_step": 1035000, "rew": 9326.6478, "rew_std": 14.85, "Agent": "TD3"}, {"env_step": 1040000, "rew": 9149.0301, "rew_std": 356.8972, "Agent": "TD3"}, {"env_step": 1045000, "rew": 9327.1727, "rew_std": 13.5036, "Agent": "TD3"}, {"env_step": 1050000, "rew": 9333.3892, "rew_std": 17.5108, "Agent": "TD3"}, {"env_step": 1055000, "rew": 8927.7454, "rew_std": 1201.0765, "Agent": "TD3"}, {"env_step": 1060000, "rew": 8845.8415, "rew_std": 1166.0041, "Agent": "TD3"}, {"env_step": 1065000, "rew": 9325.4141, "rew_std": 15.648, "Agent": "TD3"}, {"env_step": 1070000, "rew": 9319.1911, "rew_std": 14.7747, "Agent": "TD3"}, {"env_step": 1075000, "rew": 9322.9721, "rew_std": 11.5142, "Agent": "TD3"}, {"env_step": 1080000, "rew": 9321.295, "rew_std": 8.3382, "Agent": "TD3"}, {"env_step": 1085000, "rew": 8626.7788, "rew_std": 2096.5287, "Agent": "TD3"}, {"env_step": 1090000, "rew": 8702.9198, "rew_std": 1852.4358, "Agent": "TD3"}, {"env_step": 1095000, "rew": 8883.6199, "rew_std": 1310.4064, "Agent": "TD3"}, {"env_step": 1100000, "rew": 8636.9099, "rew_std": 2058.3565, "Agent": "TD3"}, {"env_step": 1105000, "rew": 9319.5485, "rew_std": 13.8536, "Agent": "TD3"}, {"env_step": 1110000, "rew": 9319.021, "rew_std": 11.4661, "Agent": "TD3"}, {"env_step": 1115000, "rew": 9319.6046, "rew_std": 11.8647, "Agent": "TD3"}, {"env_step": 1120000, "rew": 9322.6937, "rew_std": 13.0633, "Agent": "TD3"}, {"env_step": 1125000, "rew": 8474.7102, "rew_std": 2555.5902, "Agent": "TD3"}, {"env_step": 1130000, "rew": 8490.9087, "rew_std": 2063.7339, "Agent": "TD3"}, {"env_step": 1135000, "rew": 9156.6945, "rew_std": 513.5511, "Agent": "TD3"}, {"env_step": 1140000, "rew": 9323.0957, "rew_std": 17.2924, "Agent": "TD3"}, {"env_step": 1145000, "rew": 9234.5272, "rew_std": 259.9814, "Agent": "TD3"}, {"env_step": 1150000, "rew": 8800.384, "rew_std": 1114.5204, "Agent": "TD3"}, {"env_step": 1155000, "rew": 8976.3803, "rew_std": 1050.1321, "Agent": "TD3"}, {"env_step": 1160000, "rew": 9326.5404, "rew_std": 13.1945, "Agent": "TD3"}, {"env_step": 1165000, "rew": 9325.8077, "rew_std": 17.0369, "Agent": "TD3"}, {"env_step": 1170000, "rew": 8887.2947, "rew_std": 1309.6168, "Agent": "TD3"}, {"env_step": 1175000, "rew": 9322.4509, "rew_std": 17.1221, "Agent": "TD3"}, {"env_step": 1180000, "rew": 9323.5382, "rew_std": 16.3925, "Agent": "TD3"}, {"env_step": 1185000, "rew": 9325.3668, "rew_std": 17.4452, "Agent": "TD3"}, {"env_step": 1190000, "rew": 8981.4488, "rew_std": 1037.9468, "Agent": "TD3"}, {"env_step": 1195000, "rew": 8478.0402, "rew_std": 2549.7415, "Agent": "TD3"}, {"env_step": 1200000, "rew": 8589.0557, "rew_std": 2228.981, "Agent": "TD3"}, {"env_step": 1205000, "rew": 9329.7533, "rew_std": 14.2779, "Agent": "TD3"}, {"env_step": 1210000, "rew": 8508.0089, "rew_std": 2489.3719, "Agent": "TD3"}, {"env_step": 1215000, "rew": 9080.1032, "rew_std": 743.7643, "Agent": "TD3"}, {"env_step": 1220000, "rew": 8915.0462, "rew_std": 1257.401, "Agent": "TD3"}, {"env_step": 1225000, "rew": 9331.3188, "rew_std": 14.9061, "Agent": "TD3"}, {"env_step": 1230000, "rew": 9332.644, "rew_std": 15.5307, "Agent": "TD3"}, {"env_step": 1235000, "rew": 9327.1162, "rew_std": 12.8789, "Agent": "TD3"}, {"env_step": 1240000, "rew": 9318.4714, "rew_std": 7.6461, "Agent": "TD3"}, {"env_step": 1245000, "rew": 9321.1796, "rew_std": 11.7523, "Agent": "TD3"}, {"env_step": 1250000, "rew": 9326.545, "rew_std": 11.7313, "Agent": "TD3"}, {"env_step": 0, "rew": -46.1947, "rew_std": 25.5804, "Agent": "TRPO"}, {"env_step": 30720, "rew": -13.9012, "rew_std": 11.913, "Agent": "TRPO"}, {"env_step": 61440, "rew": 18.0615, "rew_std": 9.0166, "Agent": "TRPO"}, {"env_step": 92160, "rew": 51.9175, "rew_std": 16.4955, "Agent": "TRPO"}, {"env_step": 122880, "rew": 108.4958, "rew_std": 33.2241, "Agent": "TRPO"}, {"env_step": 153600, "rew": 183.9944, "rew_std": 35.8262, "Agent": "TRPO"}, {"env_step": 184320, "rew": 257.4861, "rew_std": 60.051, "Agent": "TRPO"}, {"env_step": 215040, "rew": 430.4068, "rew_std": 124.2541, "Agent": "TRPO"}, {"env_step": 245760, "rew": 564.2017, "rew_std": 146.0606, "Agent": "TRPO"}, {"env_step": 276480, "rew": 682.5168, "rew_std": 122.3187, "Agent": "TRPO"}, {"env_step": 307200, "rew": 832.5033, "rew_std": 159.2968, "Agent": "TRPO"}, {"env_step": 337920, "rew": 1014.1049, "rew_std": 221.8108, "Agent": "TRPO"}, {"env_step": 368640, "rew": 1128.5618, "rew_std": 167.9213, "Agent": "TRPO"}, {"env_step": 399360, "rew": 1146.4494, "rew_std": 215.1602, "Agent": "TRPO"}, {"env_step": 430080, "rew": 1233.6374, "rew_std": 286.8604, "Agent": "TRPO"}, {"env_step": 460800, "rew": 1261.4876, "rew_std": 301.0892, "Agent": "TRPO"}, {"env_step": 491520, "rew": 1472.6865, "rew_std": 215.7148, "Agent": "TRPO"}, {"env_step": 522240, "rew": 1501.5343, "rew_std": 428.1053, "Agent": "TRPO"}, {"env_step": 552960, "rew": 1631.5161, "rew_std": 558.1692, "Agent": "TRPO"}, {"env_step": 583680, "rew": 1741.6005, "rew_std": 483.7599, "Agent": "TRPO"}, {"env_step": 614400, "rew": 1809.3667, "rew_std": 444.8329, "Agent": "TRPO"}, {"env_step": 645120, "rew": 1943.2351, "rew_std": 386.0455, "Agent": "TRPO"}, {"env_step": 675840, "rew": 2119.3593, "rew_std": 463.9853, "Agent": "TRPO"}, {"env_step": 706560, "rew": 2160.9349, "rew_std": 299.8534, "Agent": "TRPO"}, {"env_step": 737280, "rew": 2205.6584, "rew_std": 495.8226, "Agent": "TRPO"}, {"env_step": 768000, "rew": 2148.4616, "rew_std": 617.4263, "Agent": "TRPO"}, {"env_step": 798720, "rew": 2294.2347, "rew_std": 619.654, "Agent": "TRPO"}, {"env_step": 829440, "rew": 2334.3877, "rew_std": 569.8366, "Agent": "TRPO"}, {"env_step": 860160, "rew": 2373.0195, "rew_std": 635.4872, "Agent": "TRPO"}, {"env_step": 890880, "rew": 2572.1874, "rew_std": 771.2781, "Agent": "TRPO"}, {"env_step": 921600, "rew": 2568.2581, "rew_std": 539.7726, "Agent": "TRPO"}, {"env_step": 952320, "rew": 2639.2867, "rew_std": 572.4301, "Agent": "TRPO"}, {"env_step": 983040, "rew": 2866.6508, "rew_std": 707.8937, "Agent": "TRPO"}, {"env_step": 1013760, "rew": 2731.0697, "rew_std": 675.644, "Agent": "TRPO"}, {"env_step": 1044480, "rew": 2702.1965, "rew_std": 742.4807, "Agent": "TRPO"}, {"env_step": 1075200, "rew": 3001.7621, "rew_std": 688.8314, "Agent": "TRPO"}, {"env_step": 1105920, "rew": 2877.8716, "rew_std": 922.2411, "Agent": "TRPO"}, {"env_step": 1136640, "rew": 3104.9072, "rew_std": 800.8227, "Agent": "TRPO"}, {"env_step": 1167360, "rew": 2976.5528, "rew_std": 661.5153, "Agent": "TRPO"}, {"env_step": 1198080, "rew": 3196.2228, "rew_std": 707.9679, "Agent": "TRPO"}, {"env_step": 1228800, "rew": 3237.2005, "rew_std": 505.6685, "Agent": "TRPO"}, {"env_step": 1259520, "rew": 3098.3083, "rew_std": 578.4346, "Agent": "TRPO"}, {"env_step": 1290240, "rew": 3356.962, "rew_std": 786.3838, "Agent": "TRPO"}, {"env_step": 1320960, "rew": 3241.2647, "rew_std": 600.3495, "Agent": "TRPO"}, {"env_step": 1351680, "rew": 3361.1722, "rew_std": 547.5151, "Agent": "TRPO"}, {"env_step": 1382400, "rew": 3495.3879, "rew_std": 662.5387, "Agent": "TRPO"}, {"env_step": 1413120, "rew": 3388.8443, "rew_std": 537.1016, "Agent": "TRPO"}, {"env_step": 1443840, "rew": 3493.6456, "rew_std": 509.2391, "Agent": "TRPO"}, {"env_step": 1474560, "rew": 3256.3625, "rew_std": 412.1903, "Agent": "TRPO"}, {"env_step": 1505280, "rew": 3695.3862, "rew_std": 644.5284, "Agent": "TRPO"}, {"env_step": 1536000, "rew": 3685.0253, "rew_std": 537.4812, "Agent": "TRPO"}, {"env_step": 1566720, "rew": 3565.3736, "rew_std": 647.2313, "Agent": "TRPO"}, {"env_step": 1597440, "rew": 3790.5153, "rew_std": 702.1943, "Agent": "TRPO"}, {"env_step": 1628160, "rew": 3598.3267, "rew_std": 626.2541, "Agent": "TRPO"}, {"env_step": 1658880, "rew": 3970.7287, "rew_std": 647.9836, "Agent": "TRPO"}, {"env_step": 1689600, "rew": 3658.6492, "rew_std": 645.0697, "Agent": "TRPO"}, {"env_step": 1720320, "rew": 3848.0345, "rew_std": 701.1273, "Agent": "TRPO"}, {"env_step": 1751040, "rew": 4061.4513, "rew_std": 690.0289, "Agent": "TRPO"}, {"env_step": 1781760, "rew": 3807.921, "rew_std": 706.5999, "Agent": "TRPO"}, {"env_step": 1812480, "rew": 3802.0342, "rew_std": 663.7139, "Agent": "TRPO"}, {"env_step": 1843200, "rew": 3873.6662, "rew_std": 566.3833, "Agent": "TRPO"}, {"env_step": 1873920, "rew": 4038.6082, "rew_std": 518.5079, "Agent": "TRPO"}, {"env_step": 1904640, "rew": 4167.3221, "rew_std": 440.9658, "Agent": "TRPO"}, {"env_step": 1935360, "rew": 3922.2855, "rew_std": 733.8238, "Agent": "TRPO"}, {"env_step": 1966080, "rew": 4174.7307, "rew_std": 495.1849, "Agent": "TRPO"}, {"env_step": 1996800, "rew": 4155.8384, "rew_std": 477.6376, "Agent": "TRPO"}, {"env_step": 2027520, "rew": 4259.3055, "rew_std": 577.3636, "Agent": "TRPO"}, {"env_step": 2058240, "rew": 4256.1933, "rew_std": 594.2826, "Agent": "TRPO"}, {"env_step": 2088960, "rew": 4355.0669, "rew_std": 605.8978, "Agent": "TRPO"}, {"env_step": 2119680, "rew": 4297.2252, "rew_std": 474.421, "Agent": "TRPO"}, {"env_step": 2150400, "rew": 4287.4698, "rew_std": 363.7332, "Agent": "TRPO"}, {"env_step": 2181120, "rew": 4398.2137, "rew_std": 354.3603, "Agent": "TRPO"}, {"env_step": 2211840, "rew": 4413.5796, "rew_std": 460.9681, "Agent": "TRPO"}, {"env_step": 2242560, "rew": 4329.001, "rew_std": 530.2755, "Agent": "TRPO"}, {"env_step": 2273280, "rew": 4252.3464, "rew_std": 597.5882, "Agent": "TRPO"}, {"env_step": 2304000, "rew": 4462.5657, "rew_std": 495.3802, "Agent": "TRPO"}, {"env_step": 2334720, "rew": 4374.2838, "rew_std": 457.0956, "Agent": "TRPO"}, {"env_step": 2365440, "rew": 4529.4549, "rew_std": 461.1733, "Agent": "TRPO"}, {"env_step": 2396160, "rew": 4291.4468, "rew_std": 411.2142, "Agent": "TRPO"}, {"env_step": 2426880, "rew": 4430.1459, "rew_std": 601.8812, "Agent": "TRPO"}, {"env_step": 2457600, "rew": 4508.078, "rew_std": 465.0507, "Agent": "TRPO"}, {"env_step": 2488320, "rew": 4151.3796, "rew_std": 414.656, "Agent": "TRPO"}, {"env_step": 2519040, "rew": 4705.7212, "rew_std": 477.1206, "Agent": "TRPO"}, {"env_step": 2549760, "rew": 4641.3111, "rew_std": 622.6369, "Agent": "TRPO"}, {"env_step": 2580480, "rew": 4484.1452, "rew_std": 442.0811, "Agent": "TRPO"}, {"env_step": 2611200, "rew": 4658.7294, "rew_std": 432.1496, "Agent": "TRPO"}, {"env_step": 2641920, "rew": 4625.8785, "rew_std": 357.4208, "Agent": "TRPO"}, {"env_step": 2672640, "rew": 4284.9794, "rew_std": 599.7262, "Agent": "TRPO"}, {"env_step": 2703360, "rew": 4419.1972, "rew_std": 447.5325, "Agent": "TRPO"}, {"env_step": 2734080, "rew": 4746.1067, "rew_std": 580.4515, "Agent": "TRPO"}, {"env_step": 2764800, "rew": 4756.6644, "rew_std": 502.4658, "Agent": "TRPO"}, {"env_step": 2795520, "rew": 4607.7287, "rew_std": 427.1984, "Agent": "TRPO"}, {"env_step": 2826240, "rew": 4404.4363, "rew_std": 555.3297, "Agent": "TRPO"}, {"env_step": 2856960, "rew": 4816.8623, "rew_std": 528.6794, "Agent": "TRPO"}, {"env_step": 2887680, "rew": 4753.3578, "rew_std": 715.7057, "Agent": "TRPO"}, {"env_step": 2918400, "rew": 4604.763, "rew_std": 447.4458, "Agent": "TRPO"}, {"env_step": 2949120, "rew": 4800.2693, "rew_std": 602.9667, "Agent": "TRPO"}, {"env_step": 2979840, "rew": 4772.9396, "rew_std": 368.1691, "Agent": "TRPO"}, {"env_step": 3010560, "rew": 4897.0257, "rew_std": 449.1329, "Agent": "TRPO"}, {"env_step": 3041280, "rew": 4841.7814, "rew_std": 534.8951, "Agent": "TRPO"}, {"env_step": 3072000, "rew": 4604.9043, "rew_std": 580.976, "Agent": "TRPO"}, {"env_step": 0, "rew": -46.1947, "rew_std": 25.5804, "Agent": "NPG"}, {"env_step": 30720, "rew": -29.0009, "rew_std": 15.8916, "Agent": "NPG"}, {"env_step": 61440, "rew": -10.4279, "rew_std": 11.0242, "Agent": "NPG"}, {"env_step": 92160, "rew": 11.1187, "rew_std": 9.0737, "Agent": "NPG"}, {"env_step": 122880, "rew": 27.5852, "rew_std": 16.4638, "Agent": "NPG"}, {"env_step": 153600, "rew": 56.7285, "rew_std": 23.8642, "Agent": "NPG"}, {"env_step": 184320, "rew": 81.2319, "rew_std": 29.1525, "Agent": "NPG"}, {"env_step": 215040, "rew": 159.4704, "rew_std": 47.3793, "Agent": "NPG"}, {"env_step": 245760, "rew": 174.9781, "rew_std": 57.693, "Agent": "NPG"}, {"env_step": 276480, "rew": 178.6358, "rew_std": 32.807, "Agent": "NPG"}, {"env_step": 307200, "rew": 218.3212, "rew_std": 41.035, "Agent": "NPG"}, {"env_step": 337920, "rew": 368.663, "rew_std": 80.0623, "Agent": "NPG"}, {"env_step": 368640, "rew": 432.2163, "rew_std": 63.1225, "Agent": "NPG"}, {"env_step": 399360, "rew": 568.0807, "rew_std": 125.517, "Agent": "NPG"}, {"env_step": 430080, "rew": 631.1259, "rew_std": 114.6371, "Agent": "NPG"}, {"env_step": 460800, "rew": 753.5929, "rew_std": 118.5185, "Agent": "NPG"}, {"env_step": 491520, "rew": 864.7371, "rew_std": 183.4071, "Agent": "NPG"}, {"env_step": 522240, "rew": 940.2978, "rew_std": 158.8018, "Agent": "NPG"}, {"env_step": 552960, "rew": 1021.7661, "rew_std": 159.2406, "Agent": "NPG"}, {"env_step": 583680, "rew": 1131.714, "rew_std": 150.4756, "Agent": "NPG"}, {"env_step": 614400, "rew": 1201.5181, "rew_std": 164.258, "Agent": "NPG"}, {"env_step": 645120, "rew": 1343.1191, "rew_std": 184.6686, "Agent": "NPG"}, {"env_step": 675840, "rew": 1318.401, "rew_std": 229.7468, "Agent": "NPG"}, {"env_step": 706560, "rew": 1459.1159, "rew_std": 222.6298, "Agent": "NPG"}, {"env_step": 737280, "rew": 1542.0295, "rew_std": 203.9479, "Agent": "NPG"}, {"env_step": 768000, "rew": 1634.9176, "rew_std": 218.2131, "Agent": "NPG"}, {"env_step": 798720, "rew": 1692.0983, "rew_std": 343.9785, "Agent": "NPG"}, {"env_step": 829440, "rew": 1906.1507, "rew_std": 457.5102, "Agent": "NPG"}, {"env_step": 860160, "rew": 1775.1748, "rew_std": 342.0144, "Agent": "NPG"}, {"env_step": 890880, "rew": 2102.0494, "rew_std": 355.8503, "Agent": "NPG"}, {"env_step": 921600, "rew": 2046.7794, "rew_std": 340.8884, "Agent": "NPG"}, {"env_step": 952320, "rew": 2164.3512, "rew_std": 405.7482, "Agent": "NPG"}, {"env_step": 983040, "rew": 2348.312, "rew_std": 509.9401, "Agent": "NPG"}, {"env_step": 1013760, "rew": 2365.8958, "rew_std": 523.6603, "Agent": "NPG"}, {"env_step": 1044480, "rew": 2324.9164, "rew_std": 528.4526, "Agent": "NPG"}, {"env_step": 1075200, "rew": 2493.722, "rew_std": 583.8895, "Agent": "NPG"}, {"env_step": 1105920, "rew": 2638.7282, "rew_std": 650.3356, "Agent": "NPG"}, {"env_step": 1136640, "rew": 2588.3598, "rew_std": 506.4239, "Agent": "NPG"}, {"env_step": 1167360, "rew": 2657.8218, "rew_std": 542.1361, "Agent": "NPG"}, {"env_step": 1198080, "rew": 2772.0887, "rew_std": 399.8722, "Agent": "NPG"}, {"env_step": 1228800, "rew": 2983.8532, "rew_std": 405.1427, "Agent": "NPG"}, {"env_step": 1259520, "rew": 2969.912, "rew_std": 543.0224, "Agent": "NPG"}, {"env_step": 1290240, "rew": 2898.5856, "rew_std": 687.1927, "Agent": "NPG"}, {"env_step": 1320960, "rew": 2979.687, "rew_std": 685.3181, "Agent": "NPG"}, {"env_step": 1351680, "rew": 2937.7865, "rew_std": 654.2811, "Agent": "NPG"}, {"env_step": 1382400, "rew": 3180.3262, "rew_std": 502.1675, "Agent": "NPG"}, {"env_step": 1413120, "rew": 3181.4746, "rew_std": 526.9918, "Agent": "NPG"}, {"env_step": 1443840, "rew": 3186.725, "rew_std": 483.0104, "Agent": "NPG"}, {"env_step": 1474560, "rew": 3347.1624, "rew_std": 474.3212, "Agent": "NPG"}, {"env_step": 1505280, "rew": 3264.1247, "rew_std": 542.1326, "Agent": "NPG"}, {"env_step": 1536000, "rew": 3267.3737, "rew_std": 536.477, "Agent": "NPG"}, {"env_step": 1566720, "rew": 3677.709, "rew_std": 486.0857, "Agent": "NPG"}, {"env_step": 1597440, "rew": 3379.1083, "rew_std": 683.1763, "Agent": "NPG"}, {"env_step": 1628160, "rew": 3533.6653, "rew_std": 437.9623, "Agent": "NPG"}, {"env_step": 1658880, "rew": 3534.0292, "rew_std": 787.6867, "Agent": "NPG"}, {"env_step": 1689600, "rew": 3399.7589, "rew_std": 685.1419, "Agent": "NPG"}, {"env_step": 1720320, "rew": 3587.7024, "rew_std": 471.134, "Agent": "NPG"}, {"env_step": 1751040, "rew": 3560.6233, "rew_std": 616.8093, "Agent": "NPG"}, {"env_step": 1781760, "rew": 3635.7303, "rew_std": 608.4666, "Agent": "NPG"}, {"env_step": 1812480, "rew": 3713.5899, "rew_std": 802.8577, "Agent": "NPG"}, {"env_step": 1843200, "rew": 3590.908, "rew_std": 557.7779, "Agent": "NPG"}, {"env_step": 1873920, "rew": 3903.7812, "rew_std": 655.7805, "Agent": "NPG"}, {"env_step": 1904640, "rew": 3770.5075, "rew_std": 645.9229, "Agent": "NPG"}, {"env_step": 1935360, "rew": 4110.3599, "rew_std": 657.3085, "Agent": "NPG"}, {"env_step": 1966080, "rew": 3851.8914, "rew_std": 745.2079, "Agent": "NPG"}, {"env_step": 1996800, "rew": 3998.0014, "rew_std": 650.7886, "Agent": "NPG"}, {"env_step": 2027520, "rew": 3739.3657, "rew_std": 715.6727, "Agent": "NPG"}, {"env_step": 2058240, "rew": 4230.5764, "rew_std": 578.5901, "Agent": "NPG"}, {"env_step": 2088960, "rew": 4140.935, "rew_std": 624.3245, "Agent": "NPG"}, {"env_step": 2119680, "rew": 4228.9808, "rew_std": 573.9637, "Agent": "NPG"}, {"env_step": 2150400, "rew": 4142.6649, "rew_std": 419.5781, "Agent": "NPG"}, {"env_step": 2181120, "rew": 4089.8147, "rew_std": 302.9776, "Agent": "NPG"}, {"env_step": 2211840, "rew": 4276.8854, "rew_std": 674.4424, "Agent": "NPG"}, {"env_step": 2242560, "rew": 4312.1654, "rew_std": 567.5481, "Agent": "NPG"}, {"env_step": 2273280, "rew": 4224.3566, "rew_std": 445.0954, "Agent": "NPG"}, {"env_step": 2304000, "rew": 3878.7632, "rew_std": 866.3551, "Agent": "NPG"}, {"env_step": 2334720, "rew": 4001.3801, "rew_std": 478.1705, "Agent": "NPG"}, {"env_step": 2365440, "rew": 4284.5594, "rew_std": 448.8121, "Agent": "NPG"}, {"env_step": 2396160, "rew": 4406.6923, "rew_std": 516.8683, "Agent": "NPG"}, {"env_step": 2426880, "rew": 4145.9032, "rew_std": 519.4268, "Agent": "NPG"}, {"env_step": 2457600, "rew": 4241.7632, "rew_std": 685.6791, "Agent": "NPG"}, {"env_step": 2488320, "rew": 4154.1219, "rew_std": 437.1567, "Agent": "NPG"}, {"env_step": 2519040, "rew": 4495.7374, "rew_std": 449.1806, "Agent": "NPG"}, {"env_step": 2549760, "rew": 4289.7487, "rew_std": 526.7084, "Agent": "NPG"}, {"env_step": 2580480, "rew": 4246.0211, "rew_std": 530.6265, "Agent": "NPG"}, {"env_step": 2611200, "rew": 4463.8985, "rew_std": 572.8797, "Agent": "NPG"}, {"env_step": 2641920, "rew": 4298.6282, "rew_std": 665.9416, "Agent": "NPG"}, {"env_step": 2672640, "rew": 4485.5394, "rew_std": 359.9158, "Agent": "NPG"}, {"env_step": 2703360, "rew": 4820.597, "rew_std": 318.5627, "Agent": "NPG"}, {"env_step": 2734080, "rew": 4540.1727, "rew_std": 412.9545, "Agent": "NPG"}, {"env_step": 2764800, "rew": 4500.1939, "rew_std": 596.1343, "Agent": "NPG"}, {"env_step": 2795520, "rew": 4664.0692, "rew_std": 426.6286, "Agent": "NPG"}, {"env_step": 2826240, "rew": 4516.467, "rew_std": 435.656, "Agent": "NPG"}, {"env_step": 2856960, "rew": 4545.139, "rew_std": 496.411, "Agent": "NPG"}, {"env_step": 2887680, "rew": 4444.0518, "rew_std": 596.6213, "Agent": "NPG"}, {"env_step": 2918400, "rew": 4658.8307, "rew_std": 652.0561, "Agent": "NPG"}, {"env_step": 2949120, "rew": 4509.4439, "rew_std": 466.3483, "Agent": "NPG"}, {"env_step": 2979840, "rew": 4589.8375, "rew_std": 414.0363, "Agent": "NPG"}, {"env_step": 3010560, "rew": 4693.0902, "rew_std": 490.4674, "Agent": "NPG"}, {"env_step": 3041280, "rew": 4678.8152, "rew_std": 313.0054, "Agent": "NPG"}, {"env_step": 3072000, "rew": 4736.6273, "rew_std": 376.7205, "Agent": "NPG"}, {"env_step": 0, "rew": 53.5579, "rew_std": 5.4624, "Agent": "PPO"}, {"env_step": 30720, "rew": 390.6844, "rew_std": 47.2152, "Agent": "PPO"}, {"env_step": 61440, "rew": 4440.7883, "rew_std": 1477.9099, "Agent": "PPO"}, {"env_step": 92160, "rew": 7432.4452, "rew_std": 1137.4445, "Agent": "PPO"}, {"env_step": 122880, "rew": 8723.7104, "rew_std": 840.5556, "Agent": "PPO"}, {"env_step": 153600, "rew": 8128.2175, "rew_std": 847.2373, "Agent": "PPO"}, {"env_step": 184320, "rew": 7939.7183, "rew_std": 1185.3614, "Agent": "PPO"}, {"env_step": 215040, "rew": 8215.0721, "rew_std": 1687.6094, "Agent": "PPO"}, {"env_step": 245760, "rew": 8299.0993, "rew_std": 2014.0479, "Agent": "PPO"}, {"env_step": 276480, "rew": 7574.2882, "rew_std": 1851.2534, "Agent": "PPO"}, {"env_step": 307200, "rew": 7608.4809, "rew_std": 2040.837, "Agent": "PPO"}, {"env_step": 337920, "rew": 9160.846, "rew_std": 370.1676, "Agent": "PPO"}, {"env_step": 368640, "rew": 8165.264, "rew_std": 1476.0656, "Agent": "PPO"}, {"env_step": 399360, "rew": 9231.3001, "rew_std": 270.4215, "Agent": "PPO"}, {"env_step": 430080, "rew": 8863.8433, "rew_std": 804.7622, "Agent": "PPO"}, {"env_step": 460800, "rew": 8912.687, "rew_std": 583.0749, "Agent": "PPO"}, {"env_step": 491520, "rew": 9178.6727, "rew_std": 351.559, "Agent": "PPO"}, {"env_step": 522240, "rew": 8551.0671, "rew_std": 1781.7509, "Agent": "PPO"}, {"env_step": 552960, "rew": 8041.7706, "rew_std": 1343.6888, "Agent": "PPO"}, {"env_step": 583680, "rew": 8463.9937, "rew_std": 1607.719, "Agent": "PPO"}, {"env_step": 614400, "rew": 7499.9099, "rew_std": 1967.5161, "Agent": "PPO"}, {"env_step": 645120, "rew": 7426.7158, "rew_std": 1688.5658, "Agent": "PPO"}, {"env_step": 675840, "rew": 7450.5741, "rew_std": 1562.2172, "Agent": "PPO"}, {"env_step": 706560, "rew": 7919.1652, "rew_std": 1307.948, "Agent": "PPO"}, {"env_step": 737280, "rew": 8058.6832, "rew_std": 1555.2311, "Agent": "PPO"}, {"env_step": 768000, "rew": 7796.2947, "rew_std": 1427.9864, "Agent": "PPO"}, {"env_step": 798720, "rew": 7871.2745, "rew_std": 2091.2387, "Agent": "PPO"}, {"env_step": 829440, "rew": 7788.3739, "rew_std": 1169.7356, "Agent": "PPO"}, {"env_step": 860160, "rew": 8649.8336, "rew_std": 880.0055, "Agent": "PPO"}, {"env_step": 890880, "rew": 8629.5956, "rew_std": 883.1806, "Agent": "PPO"}, {"env_step": 921600, "rew": 7053.2239, "rew_std": 1975.2603, "Agent": "PPO"}, {"env_step": 952320, "rew": 8832.0522, "rew_std": 720.8276, "Agent": "PPO"}, {"env_step": 983040, "rew": 8695.0495, "rew_std": 1114.9476, "Agent": "PPO"}, {"env_step": 1013760, "rew": 7769.8183, "rew_std": 1044.702, "Agent": "PPO"}, {"env_step": 1044480, "rew": 7672.861, "rew_std": 2120.0992, "Agent": "PPO"}, {"env_step": 1075200, "rew": 7210.2176, "rew_std": 2072.4542, "Agent": "PPO"}, {"env_step": 1105920, "rew": 6198.4025, "rew_std": 2246.6461, "Agent": "PPO"}, {"env_step": 1136640, "rew": 6235.9517, "rew_std": 2022.0933, "Agent": "PPO"}, {"env_step": 1167360, "rew": 8930.9663, "rew_std": 552.711, "Agent": "PPO"}, {"env_step": 1198080, "rew": 7589.3857, "rew_std": 2375.9742, "Agent": "PPO"}, {"env_step": 1228800, "rew": 6630.552, "rew_std": 3211.1769, "Agent": "PPO"}, {"env_step": 1259520, "rew": 6154.6071, "rew_std": 2411.6233, "Agent": "PPO"}, {"env_step": 1290240, "rew": 6570.4411, "rew_std": 2282.5933, "Agent": "PPO"}, {"env_step": 1320960, "rew": 6988.3474, "rew_std": 1774.5001, "Agent": "PPO"}, {"env_step": 1351680, "rew": 6362.3528, "rew_std": 1873.2545, "Agent": "PPO"}, {"env_step": 1382400, "rew": 8103.1912, "rew_std": 1972.6555, "Agent": "PPO"}, {"env_step": 1413120, "rew": 8173.9319, "rew_std": 1462.2099, "Agent": "PPO"}, {"env_step": 1443840, "rew": 7561.3657, "rew_std": 1242.3688, "Agent": "PPO"}, {"env_step": 1474560, "rew": 7414.5475, "rew_std": 1797.1936, "Agent": "PPO"}, {"env_step": 1505280, "rew": 7456.7353, "rew_std": 2076.4739, "Agent": "PPO"}, {"env_step": 1536000, "rew": 7188.1632, "rew_std": 1879.0908, "Agent": "PPO"}, {"env_step": 1566720, "rew": 8456.162, "rew_std": 982.2274, "Agent": "PPO"}, {"env_step": 1597440, "rew": 7070.9443, "rew_std": 2235.7345, "Agent": "PPO"}, {"env_step": 1628160, "rew": 6318.5381, "rew_std": 1411.5987, "Agent": "PPO"}, {"env_step": 1658880, "rew": 6076.6026, "rew_std": 2673.6325, "Agent": "PPO"}, {"env_step": 1689600, "rew": 7328.0759, "rew_std": 2272.6423, "Agent": "PPO"}, {"env_step": 1720320, "rew": 7814.5406, "rew_std": 2963.4102, "Agent": "PPO"}, {"env_step": 1751040, "rew": 7140.9796, "rew_std": 2355.248, "Agent": "PPO"}, {"env_step": 1781760, "rew": 6766.0236, "rew_std": 2901.3291, "Agent": "PPO"}, {"env_step": 1812480, "rew": 5966.0184, "rew_std": 2419.0426, "Agent": "PPO"}, {"env_step": 1843200, "rew": 7952.3784, "rew_std": 2461.1629, "Agent": "PPO"}, {"env_step": 1873920, "rew": 7840.1507, "rew_std": 1948.9364, "Agent": "PPO"}, {"env_step": 1904640, "rew": 6536.9612, "rew_std": 1433.0194, "Agent": "PPO"}, {"env_step": 1935360, "rew": 6992.1694, "rew_std": 2035.752, "Agent": "PPO"}, {"env_step": 1966080, "rew": 7942.765, "rew_std": 1163.106, "Agent": "PPO"}, {"env_step": 1996800, "rew": 6991.8808, "rew_std": 2000.2169, "Agent": "PPO"}, {"env_step": 2027520, "rew": 7433.3263, "rew_std": 957.6863, "Agent": "PPO"}, {"env_step": 2058240, "rew": 7583.5617, "rew_std": 1624.175, "Agent": "PPO"}, {"env_step": 2088960, "rew": 7215.9013, "rew_std": 1515.0528, "Agent": "PPO"}, {"env_step": 2119680, "rew": 5971.309, "rew_std": 2303.5213, "Agent": "PPO"}, {"env_step": 2150400, "rew": 7132.0085, "rew_std": 1651.5937, "Agent": "PPO"}, {"env_step": 2181120, "rew": 5749.5429, "rew_std": 1756.0616, "Agent": "PPO"}, {"env_step": 2211840, "rew": 7625.1017, "rew_std": 1200.5677, "Agent": "PPO"}, {"env_step": 2242560, "rew": 5903.4396, "rew_std": 1907.4404, "Agent": "PPO"}, {"env_step": 2273280, "rew": 7209.7763, "rew_std": 1078.0066, "Agent": "PPO"}, {"env_step": 2304000, "rew": 8454.0754, "rew_std": 905.2939, "Agent": "PPO"}, {"env_step": 2334720, "rew": 6988.5313, "rew_std": 1284.9775, "Agent": "PPO"}, {"env_step": 2365440, "rew": 6159.7626, "rew_std": 1692.64, "Agent": "PPO"}, {"env_step": 2396160, "rew": 7005.0111, "rew_std": 1320.0732, "Agent": "PPO"}, {"env_step": 2426880, "rew": 6899.8644, "rew_std": 1396.3963, "Agent": "PPO"}, {"env_step": 2457600, "rew": 7638.5558, "rew_std": 1618.8543, "Agent": "PPO"}, {"env_step": 2488320, "rew": 7818.5143, "rew_std": 1324.091, "Agent": "PPO"}, {"env_step": 2519040, "rew": 6183.0189, "rew_std": 2073.5887, "Agent": "PPO"}, {"env_step": 2549760, "rew": 8044.4182, "rew_std": 922.0843, "Agent": "PPO"}, {"env_step": 2580480, "rew": 8066.0202, "rew_std": 1208.6707, "Agent": "PPO"}, {"env_step": 2611200, "rew": 7911.9955, "rew_std": 1461.3701, "Agent": "PPO"}, {"env_step": 2641920, "rew": 7744.7357, "rew_std": 1843.5026, "Agent": "PPO"}, {"env_step": 2672640, "rew": 7727.1222, "rew_std": 1227.0877, "Agent": "PPO"}, {"env_step": 2703360, "rew": 6815.0618, "rew_std": 1496.2736, "Agent": "PPO"}, {"env_step": 2734080, "rew": 6665.7163, "rew_std": 1282.2237, "Agent": "PPO"}, {"env_step": 2764800, "rew": 6194.4134, "rew_std": 1379.2422, "Agent": "PPO"}, {"env_step": 2795520, "rew": 7207.3626, "rew_std": 935.8931, "Agent": "PPO"}, {"env_step": 2826240, "rew": 8002.2676, "rew_std": 1064.0059, "Agent": "PPO"}, {"env_step": 2856960, "rew": 7120.8058, "rew_std": 1433.227, "Agent": "PPO"}, {"env_step": 2887680, "rew": 6511.8525, "rew_std": 1660.6839, "Agent": "PPO"}, {"env_step": 2918400, "rew": 6734.165, "rew_std": 1341.1131, "Agent": "PPO"}, {"env_step": 2949120, "rew": 6980.5115, "rew_std": 1604.3832, "Agent": "PPO"}, {"env_step": 2979840, "rew": 6470.7865, "rew_std": 1530.6075, "Agent": "PPO"}, {"env_step": 3010560, "rew": 7347.2895, "rew_std": 1853.1296, "Agent": "PPO"}, {"env_step": 3041280, "rew": 7558.1526, "rew_std": 1060.3463, "Agent": "PPO"}, {"env_step": 3072000, "rew": 7189.4355, "rew_std": 2075.9112, "Agent": "PPO"}, {"env_step": 0, "rew": 60.0108, "rew_std": 6.0382, "Agent": "REINFORCE"}, {"env_step": 30720, "rew": 66.8851, "rew_std": 4.8886, "Agent": "REINFORCE"}, {"env_step": 61440, "rew": 82.7432, "rew_std": 7.4298, "Agent": "REINFORCE"}, {"env_step": 92160, "rew": 96.5871, "rew_std": 14.5135, "Agent": "REINFORCE"}, {"env_step": 122880, "rew": 112.431, "rew_std": 7.34, "Agent": "REINFORCE"}, {"env_step": 153600, "rew": 122.4389, "rew_std": 18.2545, "Agent": "REINFORCE"}, {"env_step": 184320, "rew": 155.4821, "rew_std": 34.0244, "Agent": "REINFORCE"}, {"env_step": 215040, "rew": 183.7388, "rew_std": 29.0268, "Agent": "REINFORCE"}, {"env_step": 245760, "rew": 228.9308, "rew_std": 42.0714, "Agent": "REINFORCE"}, {"env_step": 276480, "rew": 295.3232, "rew_std": 37.5235, "Agent": "REINFORCE"}, {"env_step": 307200, "rew": 290.0857, "rew_std": 26.9366, "Agent": "REINFORCE"}, {"env_step": 337920, "rew": 327.8566, "rew_std": 61.7201, "Agent": "REINFORCE"}, {"env_step": 368640, "rew": 357.6878, "rew_std": 41.0322, "Agent": "REINFORCE"}, {"env_step": 399360, "rew": 366.3419, "rew_std": 33.2445, "Agent": "REINFORCE"}, {"env_step": 430080, "rew": 381.9601, "rew_std": 66.8866, "Agent": "REINFORCE"}, {"env_step": 460800, "rew": 368.2484, "rew_std": 48.6524, "Agent": "REINFORCE"}, {"env_step": 491520, "rew": 378.9139, "rew_std": 37.3082, "Agent": "REINFORCE"}, {"env_step": 522240, "rew": 380.7339, "rew_std": 55.8626, "Agent": "REINFORCE"}, {"env_step": 552960, "rew": 387.7164, "rew_std": 34.7361, "Agent": "REINFORCE"}, {"env_step": 583680, "rew": 376.7374, "rew_std": 31.788, "Agent": "REINFORCE"}, {"env_step": 614400, "rew": 353.7732, "rew_std": 37.8586, "Agent": "REINFORCE"}, {"env_step": 645120, "rew": 357.7262, "rew_std": 68.3217, "Agent": "REINFORCE"}, {"env_step": 675840, "rew": 370.7562, "rew_std": 67.7155, "Agent": "REINFORCE"}, {"env_step": 706560, "rew": 383.0227, "rew_std": 54.1512, "Agent": "REINFORCE"}, {"env_step": 737280, "rew": 368.886, "rew_std": 56.9815, "Agent": "REINFORCE"}, {"env_step": 768000, "rew": 404.9809, "rew_std": 47.8215, "Agent": "REINFORCE"}, {"env_step": 798720, "rew": 426.8464, "rew_std": 50.1806, "Agent": "REINFORCE"}, {"env_step": 829440, "rew": 370.5083, "rew_std": 42.005, "Agent": "REINFORCE"}, {"env_step": 860160, "rew": 436.8902, "rew_std": 60.5663, "Agent": "REINFORCE"}, {"env_step": 890880, "rew": 420.3315, "rew_std": 82.9773, "Agent": "REINFORCE"}, {"env_step": 921600, "rew": 382.7165, "rew_std": 64.6931, "Agent": "REINFORCE"}, {"env_step": 952320, "rew": 424.8106, "rew_std": 55.993, "Agent": "REINFORCE"}, {"env_step": 983040, "rew": 434.6915, "rew_std": 56.8445, "Agent": "REINFORCE"}, {"env_step": 1013760, "rew": 427.4304, "rew_std": 54.1882, "Agent": "REINFORCE"}, {"env_step": 1044480, "rew": 435.5741, "rew_std": 80.3884, "Agent": "REINFORCE"}, {"env_step": 1075200, "rew": 417.8789, "rew_std": 48.7039, "Agent": "REINFORCE"}, {"env_step": 1105920, "rew": 388.0945, "rew_std": 69.9208, "Agent": "REINFORCE"}, {"env_step": 1136640, "rew": 416.8366, "rew_std": 48.7602, "Agent": "REINFORCE"}, {"env_step": 1167360, "rew": 490.229, "rew_std": 59.7515, "Agent": "REINFORCE"}, {"env_step": 1198080, "rew": 416.1175, "rew_std": 39.6707, "Agent": "REINFORCE"}, {"env_step": 1228800, "rew": 422.501, "rew_std": 63.4279, "Agent": "REINFORCE"}, {"env_step": 1259520, "rew": 445.428, "rew_std": 79.5721, "Agent": "REINFORCE"}, {"env_step": 1290240, "rew": 484.8554, "rew_std": 54.9234, "Agent": "REINFORCE"}, {"env_step": 1320960, "rew": 405.2333, "rew_std": 64.6929, "Agent": "REINFORCE"}, {"env_step": 1351680, "rew": 439.0572, "rew_std": 77.0379, "Agent": "REINFORCE"}, {"env_step": 1382400, "rew": 413.4107, "rew_std": 64.2411, "Agent": "REINFORCE"}, {"env_step": 1413120, "rew": 477.1185, "rew_std": 70.4266, "Agent": "REINFORCE"}, {"env_step": 1443840, "rew": 437.3873, "rew_std": 65.7348, "Agent": "REINFORCE"}, {"env_step": 1474560, "rew": 508.2686, "rew_std": 53.6442, "Agent": "REINFORCE"}, {"env_step": 1505280, "rew": 431.5613, "rew_std": 58.691, "Agent": "REINFORCE"}, {"env_step": 1536000, "rew": 504.4616, "rew_std": 104.7558, "Agent": "REINFORCE"}, {"env_step": 1566720, "rew": 514.0627, "rew_std": 113.1402, "Agent": "REINFORCE"}, {"env_step": 1597440, "rew": 438.4226, "rew_std": 85.4795, "Agent": "REINFORCE"}, {"env_step": 1628160, "rew": 508.8552, "rew_std": 73.1902, "Agent": "REINFORCE"}, {"env_step": 1658880, "rew": 524.1776, "rew_std": 46.6835, "Agent": "REINFORCE"}, {"env_step": 1689600, "rew": 443.152, "rew_std": 72.039, "Agent": "REINFORCE"}, {"env_step": 1720320, "rew": 514.0757, "rew_std": 102.6964, "Agent": "REINFORCE"}, {"env_step": 1751040, "rew": 536.2226, "rew_std": 100.2774, "Agent": "REINFORCE"}, {"env_step": 1781760, "rew": 512.4202, "rew_std": 69.555, "Agent": "REINFORCE"}, {"env_step": 1812480, "rew": 466.9226, "rew_std": 51.3613, "Agent": "REINFORCE"}, {"env_step": 1843200, "rew": 526.6333, "rew_std": 110.1826, "Agent": "REINFORCE"}, {"env_step": 1873920, "rew": 563.423, "rew_std": 88.7347, "Agent": "REINFORCE"}, {"env_step": 1904640, "rew": 488.7421, "rew_std": 94.6149, "Agent": "REINFORCE"}, {"env_step": 1935360, "rew": 527.4116, "rew_std": 72.0684, "Agent": "REINFORCE"}, {"env_step": 1966080, "rew": 584.751, "rew_std": 106.5506, "Agent": "REINFORCE"}, {"env_step": 1996800, "rew": 512.967, "rew_std": 76.5018, "Agent": "REINFORCE"}, {"env_step": 2027520, "rew": 520.3164, "rew_std": 79.305, "Agent": "REINFORCE"}, {"env_step": 2058240, "rew": 543.088, "rew_std": 180.1451, "Agent": "REINFORCE"}, {"env_step": 2088960, "rew": 572.2448, "rew_std": 112.2051, "Agent": "REINFORCE"}, {"env_step": 2119680, "rew": 483.6695, "rew_std": 79.7298, "Agent": "REINFORCE"}, {"env_step": 2150400, "rew": 576.0319, "rew_std": 166.6031, "Agent": "REINFORCE"}, {"env_step": 2181120, "rew": 541.9739, "rew_std": 104.8883, "Agent": "REINFORCE"}, {"env_step": 2211840, "rew": 584.6133, "rew_std": 134.5258, "Agent": "REINFORCE"}, {"env_step": 2242560, "rew": 571.4616, "rew_std": 102.7021, "Agent": "REINFORCE"}, {"env_step": 2273280, "rew": 695.0859, "rew_std": 210.1589, "Agent": "REINFORCE"}, {"env_step": 2304000, "rew": 622.2008, "rew_std": 153.9995, "Agent": "REINFORCE"}, {"env_step": 2334720, "rew": 599.7726, "rew_std": 122.7385, "Agent": "REINFORCE"}, {"env_step": 2365440, "rew": 649.5761, "rew_std": 106.0541, "Agent": "REINFORCE"}, {"env_step": 2396160, "rew": 634.764, "rew_std": 102.9666, "Agent": "REINFORCE"}, {"env_step": 2426880, "rew": 647.247, "rew_std": 46.9642, "Agent": "REINFORCE"}, {"env_step": 2457600, "rew": 619.0498, "rew_std": 100.3413, "Agent": "REINFORCE"}, {"env_step": 2488320, "rew": 669.8866, "rew_std": 140.8498, "Agent": "REINFORCE"}, {"env_step": 2519040, "rew": 662.3441, "rew_std": 140.1168, "Agent": "REINFORCE"}, {"env_step": 2549760, "rew": 810.179, "rew_std": 110.0032, "Agent": "REINFORCE"}, {"env_step": 2580480, "rew": 725.5527, "rew_std": 201.8621, "Agent": "REINFORCE"}, {"env_step": 2611200, "rew": 698.9872, "rew_std": 191.0236, "Agent": "REINFORCE"}, {"env_step": 2641920, "rew": 767.5747, "rew_std": 260.1571, "Agent": "REINFORCE"}, {"env_step": 2672640, "rew": 753.5526, "rew_std": 214.4276, "Agent": "REINFORCE"}, {"env_step": 2703360, "rew": 666.5184, "rew_std": 142.1623, "Agent": "REINFORCE"}, {"env_step": 2734080, "rew": 755.6162, "rew_std": 171.7469, "Agent": "REINFORCE"}, {"env_step": 2764800, "rew": 802.2893, "rew_std": 151.2001, "Agent": "REINFORCE"}, {"env_step": 2795520, "rew": 828.3906, "rew_std": 168.4727, "Agent": "REINFORCE"}, {"env_step": 2826240, "rew": 859.1231, "rew_std": 185.7303, "Agent": "REINFORCE"}, {"env_step": 2856960, "rew": 847.9353, "rew_std": 255.6323, "Agent": "REINFORCE"}, {"env_step": 2887680, "rew": 848.9568, "rew_std": 255.9742, "Agent": "REINFORCE"}, {"env_step": 2918400, "rew": 980.2406, "rew_std": 319.6858, "Agent": "REINFORCE"}, {"env_step": 2949120, "rew": 948.7681, "rew_std": 169.0014, "Agent": "REINFORCE"}, {"env_step": 2979840, "rew": 1016.5807, "rew_std": 262.9115, "Agent": "REINFORCE"}, {"env_step": 3010560, "rew": 1019.1342, "rew_std": 297.7281, "Agent": "REINFORCE"}, {"env_step": 3041280, "rew": 1059.747, "rew_std": 307.6851, "Agent": "REINFORCE"}, {"env_step": 3072000, "rew": 978.2747, "rew_std": 345.3194, "Agent": "REINFORCE"}, {"env_step": 3102720, "rew": 1160.4319, "rew_std": 464.3403, "Agent": "REINFORCE"}, {"env_step": 3133440, "rew": 1200.1039, "rew_std": 362.0233, "Agent": "REINFORCE"}, {"env_step": 3164160, "rew": 1059.5822, "rew_std": 280.4559, "Agent": "REINFORCE"}, {"env_step": 3194880, "rew": 1225.6168, "rew_std": 371.406, "Agent": "REINFORCE"}, {"env_step": 3225600, "rew": 1282.9805, "rew_std": 302.5772, "Agent": "REINFORCE"}, {"env_step": 3256320, "rew": 1273.2309, "rew_std": 349.0168, "Agent": "REINFORCE"}, {"env_step": 3287040, "rew": 1300.563, "rew_std": 425.1841, "Agent": "REINFORCE"}, {"env_step": 3317760, "rew": 1011.3652, "rew_std": 267.331, "Agent": "REINFORCE"}, {"env_step": 3348480, "rew": 1282.4909, "rew_std": 294.3243, "Agent": "REINFORCE"}, {"env_step": 3379200, "rew": 1185.7562, "rew_std": 321.7863, "Agent": "REINFORCE"}, {"env_step": 3409920, "rew": 1171.4334, "rew_std": 360.5962, "Agent": "REINFORCE"}, {"env_step": 3440640, "rew": 1195.9001, "rew_std": 331.9413, "Agent": "REINFORCE"}, {"env_step": 3471360, "rew": 1312.6145, "rew_std": 329.9906, "Agent": "REINFORCE"}, {"env_step": 3502080, "rew": 1415.1494, "rew_std": 404.2971, "Agent": "REINFORCE"}, {"env_step": 3532800, "rew": 1402.2373, "rew_std": 322.0836, "Agent": "REINFORCE"}, {"env_step": 3563520, "rew": 1508.4332, "rew_std": 578.56, "Agent": "REINFORCE"}, {"env_step": 3594240, "rew": 1398.8295, "rew_std": 532.5828, "Agent": "REINFORCE"}, {"env_step": 3624960, "rew": 1158.1128, "rew_std": 349.4831, "Agent": "REINFORCE"}, {"env_step": 3655680, "rew": 1386.2969, "rew_std": 574.8848, "Agent": "REINFORCE"}, {"env_step": 3686400, "rew": 1214.1347, "rew_std": 394.7353, "Agent": "REINFORCE"}, {"env_step": 3717120, "rew": 1749.9282, "rew_std": 1070.3417, "Agent": "REINFORCE"}, {"env_step": 3747840, "rew": 1778.2408, "rew_std": 830.1728, "Agent": "REINFORCE"}, {"env_step": 3778560, "rew": 1788.488, "rew_std": 940.4839, "Agent": "REINFORCE"}, {"env_step": 3809280, "rew": 1806.6416, "rew_std": 694.6308, "Agent": "REINFORCE"}, {"env_step": 3840000, "rew": 1628.9717, "rew_std": 993.5847, "Agent": "REINFORCE"}, {"env_step": 3870720, "rew": 1612.6756, "rew_std": 955.6134, "Agent": "REINFORCE"}, {"env_step": 3901440, "rew": 1760.4958, "rew_std": 1022.6508, "Agent": "REINFORCE"}, {"env_step": 3932160, "rew": 1685.1773, "rew_std": 963.1458, "Agent": "REINFORCE"}, {"env_step": 3962880, "rew": 2038.9305, "rew_std": 873.6692, "Agent": "REINFORCE"}, {"env_step": 3993600, "rew": 1994.5574, "rew_std": 874.5238, "Agent": "REINFORCE"}, {"env_step": 4024320, "rew": 2290.5792, "rew_std": 893.5474, "Agent": "REINFORCE"}, {"env_step": 4055040, "rew": 2093.8009, "rew_std": 907.8444, "Agent": "REINFORCE"}, {"env_step": 4085760, "rew": 1935.4176, "rew_std": 949.8348, "Agent": "REINFORCE"}, {"env_step": 4116480, "rew": 1591.5349, "rew_std": 572.0568, "Agent": "REINFORCE"}, {"env_step": 4147200, "rew": 2063.1639, "rew_std": 1069.8277, "Agent": "REINFORCE"}, {"env_step": 4177920, "rew": 2142.1858, "rew_std": 856.4905, "Agent": "REINFORCE"}, {"env_step": 4208640, "rew": 2110.9033, "rew_std": 669.7878, "Agent": "REINFORCE"}, {"env_step": 4239360, "rew": 1877.9988, "rew_std": 973.6438, "Agent": "REINFORCE"}, {"env_step": 4270080, "rew": 2330.0562, "rew_std": 1048.8469, "Agent": "REINFORCE"}, {"env_step": 4300800, "rew": 2753.4828, "rew_std": 1178.0501, "Agent": "REINFORCE"}, {"env_step": 4331520, "rew": 2482.2992, "rew_std": 786.2175, "Agent": "REINFORCE"}, {"env_step": 4362240, "rew": 2500.011, "rew_std": 857.6858, "Agent": "REINFORCE"}, {"env_step": 4392960, "rew": 2082.2058, "rew_std": 875.8882, "Agent": "REINFORCE"}, {"env_step": 4423680, "rew": 2695.8844, "rew_std": 1156.2615, "Agent": "REINFORCE"}, {"env_step": 4454400, "rew": 2012.0529, "rew_std": 639.3927, "Agent": "REINFORCE"}, {"env_step": 4485120, "rew": 2192.864, "rew_std": 1125.1998, "Agent": "REINFORCE"}, {"env_step": 4515840, "rew": 3015.4809, "rew_std": 1806.7113, "Agent": "REINFORCE"}, {"env_step": 4546560, "rew": 2323.9941, "rew_std": 547.3481, "Agent": "REINFORCE"}, {"env_step": 4577280, "rew": 2732.3637, "rew_std": 1304.811, "Agent": "REINFORCE"}, {"env_step": 4608000, "rew": 2099.5867, "rew_std": 709.0763, "Agent": "REINFORCE"}, {"env_step": 4638720, "rew": 2504.8862, "rew_std": 1158.8266, "Agent": "REINFORCE"}, {"env_step": 4669440, "rew": 2336.4626, "rew_std": 1013.3587, "Agent": "REINFORCE"}, {"env_step": 4700160, "rew": 2820.9975, "rew_std": 700.505, "Agent": "REINFORCE"}, {"env_step": 4730880, "rew": 2551.0595, "rew_std": 751.2494, "Agent": "REINFORCE"}, {"env_step": 4761600, "rew": 2306.5164, "rew_std": 725.9532, "Agent": "REINFORCE"}, {"env_step": 4792320, "rew": 2176.1637, "rew_std": 741.4824, "Agent": "REINFORCE"}, {"env_step": 4823040, "rew": 2966.1284, "rew_std": 1269.1187, "Agent": "REINFORCE"}, {"env_step": 4853760, "rew": 3351.2094, "rew_std": 1112.0173, "Agent": "REINFORCE"}, {"env_step": 4884480, "rew": 3842.1464, "rew_std": 1570.3156, "Agent": "REINFORCE"}, {"env_step": 4915200, "rew": 2989.541, "rew_std": 1266.7222, "Agent": "REINFORCE"}, {"env_step": 4945920, "rew": 2758.6786, "rew_std": 990.1936, "Agent": "REINFORCE"}, {"env_step": 4976640, "rew": 3405.5566, "rew_std": 1518.9021, "Agent": "REINFORCE"}, {"env_step": 5007360, "rew": 3869.2983, "rew_std": 1851.353, "Agent": "REINFORCE"}, {"env_step": 5038080, "rew": 2729.3042, "rew_std": 892.573, "Agent": "REINFORCE"}, {"env_step": 5068800, "rew": 3314.1694, "rew_std": 1323.7672, "Agent": "REINFORCE"}, {"env_step": 5099520, "rew": 4038.4395, "rew_std": 943.1128, "Agent": "REINFORCE"}, {"env_step": 5130240, "rew": 3256.4594, "rew_std": 1306.8482, "Agent": "REINFORCE"}, {"env_step": 5160960, "rew": 2987.736, "rew_std": 839.0792, "Agent": "REINFORCE"}, {"env_step": 5191680, "rew": 3309.6143, "rew_std": 1086.4379, "Agent": "REINFORCE"}, {"env_step": 5222400, "rew": 3487.2926, "rew_std": 1349.5377, "Agent": "REINFORCE"}, {"env_step": 5253120, "rew": 3662.9314, "rew_std": 966.6369, "Agent": "REINFORCE"}, {"env_step": 5283840, "rew": 3951.659, "rew_std": 1427.5392, "Agent": "REINFORCE"}, {"env_step": 5314560, "rew": 3243.6001, "rew_std": 1070.5047, "Agent": "REINFORCE"}, {"env_step": 5345280, "rew": 3722.7722, "rew_std": 1532.4533, "Agent": "REINFORCE"}, {"env_step": 5376000, "rew": 3645.6966, "rew_std": 1085.0242, "Agent": "REINFORCE"}, {"env_step": 5406720, "rew": 3553.8285, "rew_std": 1506.3794, "Agent": "REINFORCE"}, {"env_step": 5437440, "rew": 3428.5119, "rew_std": 1508.2119, "Agent": "REINFORCE"}, {"env_step": 5468160, "rew": 3170.9381, "rew_std": 1967.1882, "Agent": "REINFORCE"}, {"env_step": 5498880, "rew": 3263.9317, "rew_std": 1140.5613, "Agent": "REINFORCE"}, {"env_step": 5529600, "rew": 2956.3149, "rew_std": 960.4775, "Agent": "REINFORCE"}, {"env_step": 5560320, "rew": 3669.0042, "rew_std": 1323.1323, "Agent": "REINFORCE"}, {"env_step": 5591040, "rew": 4026.5697, "rew_std": 1166.8394, "Agent": "REINFORCE"}, {"env_step": 5621760, "rew": 4183.9457, "rew_std": 1343.1734, "Agent": "REINFORCE"}, {"env_step": 5652480, "rew": 4219.2541, "rew_std": 968.9301, "Agent": "REINFORCE"}, {"env_step": 5683200, "rew": 3545.1333, "rew_std": 1469.0964, "Agent": "REINFORCE"}, {"env_step": 5713920, "rew": 3831.7193, "rew_std": 1692.2045, "Agent": "REINFORCE"}, {"env_step": 5744640, "rew": 3304.0221, "rew_std": 1336.8871, "Agent": "REINFORCE"}, {"env_step": 5775360, "rew": 3031.8385, "rew_std": 1379.532, "Agent": "REINFORCE"}, {"env_step": 5806080, "rew": 3934.2707, "rew_std": 889.5228, "Agent": "REINFORCE"}, {"env_step": 5836800, "rew": 3236.5811, "rew_std": 1150.3347, "Agent": "REINFORCE"}, {"env_step": 5867520, "rew": 3755.1395, "rew_std": 1577.0949, "Agent": "REINFORCE"}, {"env_step": 5898240, "rew": 4502.6478, "rew_std": 1060.7047, "Agent": "REINFORCE"}, {"env_step": 5928960, "rew": 4059.5553, "rew_std": 1356.9739, "Agent": "REINFORCE"}, {"env_step": 5959680, "rew": 4304.8758, "rew_std": 1531.4219, "Agent": "REINFORCE"}, {"env_step": 5990400, "rew": 4732.0622, "rew_std": 1401.7171, "Agent": "REINFORCE"}, {"env_step": 6021120, "rew": 4927.5947, "rew_std": 1735.7823, "Agent": "REINFORCE"}, {"env_step": 6051840, "rew": 4896.0143, "rew_std": 2095.2379, "Agent": "REINFORCE"}, {"env_step": 6082560, "rew": 4038.2881, "rew_std": 2124.3119, "Agent": "REINFORCE"}, {"env_step": 6113280, "rew": 4884.944, "rew_std": 2155.9927, "Agent": "REINFORCE"}, {"env_step": 6144000, "rew": 4460.7201, "rew_std": 1099.7811, "Agent": "REINFORCE"}, {"env_step": 6174720, "rew": 4355.8913, "rew_std": 994.335, "Agent": "REINFORCE"}, {"env_step": 6205440, "rew": 4483.3853, "rew_std": 1632.5418, "Agent": "REINFORCE"}, {"env_step": 6236160, "rew": 5376.8727, "rew_std": 1175.4768, "Agent": "REINFORCE"}, {"env_step": 6266880, "rew": 5028.3366, "rew_std": 1725.2833, "Agent": "REINFORCE"}, {"env_step": 6297600, "rew": 5834.3314, "rew_std": 1182.5593, "Agent": "REINFORCE"}, {"env_step": 6328320, "rew": 5169.7081, "rew_std": 1486.7668, "Agent": "REINFORCE"}, {"env_step": 6359040, "rew": 5154.2478, "rew_std": 1111.2708, "Agent": "REINFORCE"}, {"env_step": 6389760, "rew": 4502.9293, "rew_std": 1125.813, "Agent": "REINFORCE"}, {"env_step": 6420480, "rew": 4459.1707, "rew_std": 1108.9785, "Agent": "REINFORCE"}, {"env_step": 6451200, "rew": 5311.4787, "rew_std": 1873.4583, "Agent": "REINFORCE"}, {"env_step": 6481920, "rew": 5532.3912, "rew_std": 2139.4747, "Agent": "REINFORCE"}, {"env_step": 6512640, "rew": 5280.449, "rew_std": 1447.4049, "Agent": "REINFORCE"}, {"env_step": 6543360, "rew": 5015.1148, "rew_std": 1553.0882, "Agent": "REINFORCE"}, {"env_step": 6574080, "rew": 4470.1792, "rew_std": 1804.0072, "Agent": "REINFORCE"}, {"env_step": 6604800, "rew": 4214.2721, "rew_std": 1353.1881, "Agent": "REINFORCE"}, {"env_step": 6635520, "rew": 5256.508, "rew_std": 1465.1617, "Agent": "REINFORCE"}, {"env_step": 6666240, "rew": 5192.832, "rew_std": 1256.6718, "Agent": "REINFORCE"}, {"env_step": 6696960, "rew": 5149.1533, "rew_std": 1938.2363, "Agent": "REINFORCE"}, {"env_step": 6727680, "rew": 4871.797, "rew_std": 1085.56, "Agent": "REINFORCE"}, {"env_step": 6758400, "rew": 5401.9239, "rew_std": 1134.1753, "Agent": "REINFORCE"}, {"env_step": 6789120, "rew": 4990.7386, "rew_std": 1913.7971, "Agent": "REINFORCE"}, {"env_step": 6819840, "rew": 5715.5955, "rew_std": 1215.5666, "Agent": "REINFORCE"}, {"env_step": 6850560, "rew": 5857.0221, "rew_std": 1281.9894, "Agent": "REINFORCE"}, {"env_step": 6881280, "rew": 5648.1819, "rew_std": 1136.7722, "Agent": "REINFORCE"}, {"env_step": 6912000, "rew": 5817.961, "rew_std": 1547.1178, "Agent": "REINFORCE"}, {"env_step": 6942720, "rew": 6049.0488, "rew_std": 1193.2055, "Agent": "REINFORCE"}, {"env_step": 6973440, "rew": 5471.947, "rew_std": 1142.1615, "Agent": "REINFORCE"}, {"env_step": 7004160, "rew": 6081.4353, "rew_std": 1233.8771, "Agent": "REINFORCE"}, {"env_step": 7034880, "rew": 5363.5289, "rew_std": 970.5114, "Agent": "REINFORCE"}, {"env_step": 7065600, "rew": 5779.5069, "rew_std": 1246.4714, "Agent": "REINFORCE"}, {"env_step": 7096320, "rew": 5788.5086, "rew_std": 1228.3144, "Agent": "REINFORCE"}, {"env_step": 7127040, "rew": 4967.2815, "rew_std": 1767.1772, "Agent": "REINFORCE"}, {"env_step": 7157760, "rew": 4538.9419, "rew_std": 1589.6502, "Agent": "REINFORCE"}, {"env_step": 7188480, "rew": 5137.0493, "rew_std": 1700.3284, "Agent": "REINFORCE"}, {"env_step": 7219200, "rew": 5012.8316, "rew_std": 1527.0919, "Agent": "REINFORCE"}, {"env_step": 7249920, "rew": 6073.756, "rew_std": 1803.8926, "Agent": "REINFORCE"}, {"env_step": 7280640, "rew": 5434.4797, "rew_std": 1784.3038, "Agent": "REINFORCE"}, {"env_step": 7311360, "rew": 5634.7391, "rew_std": 1627.405, "Agent": "REINFORCE"}, {"env_step": 7342080, "rew": 6050.9446, "rew_std": 1251.8911, "Agent": "REINFORCE"}, {"env_step": 7372800, "rew": 5799.2364, "rew_std": 1298.4554, "Agent": "REINFORCE"}, {"env_step": 7403520, "rew": 5395.9884, "rew_std": 1760.5917, "Agent": "REINFORCE"}, {"env_step": 7434240, "rew": 5073.6754, "rew_std": 1554.3566, "Agent": "REINFORCE"}, {"env_step": 7464960, "rew": 5815.9828, "rew_std": 1015.2138, "Agent": "REINFORCE"}, {"env_step": 7495680, "rew": 5884.6845, "rew_std": 1499.5421, "Agent": "REINFORCE"}, {"env_step": 7526400, "rew": 6234.7842, "rew_std": 1264.545, "Agent": "REINFORCE"}, {"env_step": 7557120, "rew": 5556.2612, "rew_std": 1266.8728, "Agent": "REINFORCE"}, {"env_step": 7587840, "rew": 5691.0096, "rew_std": 1742.2202, "Agent": "REINFORCE"}, {"env_step": 7618560, "rew": 5424.3148, "rew_std": 1324.0728, "Agent": "REINFORCE"}, {"env_step": 7649280, "rew": 5504.6508, "rew_std": 1257.7258, "Agent": "REINFORCE"}, {"env_step": 7680000, "rew": 5705.7458, "rew_std": 1287.1892, "Agent": "REINFORCE"}, {"env_step": 7710720, "rew": 5963.5763, "rew_std": 1430.009, "Agent": "REINFORCE"}, {"env_step": 7741440, "rew": 5339.8363, "rew_std": 1447.4924, "Agent": "REINFORCE"}, {"env_step": 7772160, "rew": 5916.8767, "rew_std": 1510.3091, "Agent": "REINFORCE"}, {"env_step": 7802880, "rew": 5386.2398, "rew_std": 1841.1415, "Agent": "REINFORCE"}, {"env_step": 7833600, "rew": 5538.2181, "rew_std": 1806.8538, "Agent": "REINFORCE"}, {"env_step": 7864320, "rew": 5652.5788, "rew_std": 1777.7795, "Agent": "REINFORCE"}, {"env_step": 7895040, "rew": 6067.3995, "rew_std": 1497.3349, "Agent": "REINFORCE"}, {"env_step": 7925760, "rew": 6354.8653, "rew_std": 1497.8889, "Agent": "REINFORCE"}, {"env_step": 7956480, "rew": 5823.1186, "rew_std": 1269.9062, "Agent": "REINFORCE"}, {"env_step": 7987200, "rew": 6275.2123, "rew_std": 1495.5294, "Agent": "REINFORCE"}, {"env_step": 8017920, "rew": 6120.7756, "rew_std": 1372.5631, "Agent": "REINFORCE"}, {"env_step": 8048640, "rew": 6376.4303, "rew_std": 1571.4903, "Agent": "REINFORCE"}, {"env_step": 8079360, "rew": 6652.7371, "rew_std": 1835.958, "Agent": "REINFORCE"}, {"env_step": 8110080, "rew": 6444.4723, "rew_std": 1110.3799, "Agent": "REINFORCE"}, {"env_step": 8140800, "rew": 7217.0162, "rew_std": 1063.6642, "Agent": "REINFORCE"}, {"env_step": 8171520, "rew": 5653.2569, "rew_std": 1414.6232, "Agent": "REINFORCE"}, {"env_step": 8202240, "rew": 6460.5066, "rew_std": 952.3641, "Agent": "REINFORCE"}, {"env_step": 8232960, "rew": 5945.2504, "rew_std": 1054.7127, "Agent": "REINFORCE"}, {"env_step": 8263680, "rew": 5392.9994, "rew_std": 946.8632, "Agent": "REINFORCE"}, {"env_step": 8294400, "rew": 5711.6941, "rew_std": 1424.4968, "Agent": "REINFORCE"}, {"env_step": 8325120, "rew": 6544.5364, "rew_std": 1623.5174, "Agent": "REINFORCE"}, {"env_step": 8355840, "rew": 5464.2413, "rew_std": 2142.0475, "Agent": "REINFORCE"}, {"env_step": 8386560, "rew": 6105.3293, "rew_std": 2194.9652, "Agent": "REINFORCE"}, {"env_step": 8417280, "rew": 5271.8896, "rew_std": 1588.2101, "Agent": "REINFORCE"}, {"env_step": 8448000, "rew": 5453.34, "rew_std": 1647.1389, "Agent": "REINFORCE"}, {"env_step": 8478720, "rew": 5382.6643, "rew_std": 1333.2283, "Agent": "REINFORCE"}, {"env_step": 8509440, "rew": 6127.8232, "rew_std": 1471.4567, "Agent": "REINFORCE"}, {"env_step": 8540160, "rew": 5988.2806, "rew_std": 1687.2463, "Agent": "REINFORCE"}, {"env_step": 8570880, "rew": 6061.9376, "rew_std": 954.6449, "Agent": "REINFORCE"}, {"env_step": 8601600, "rew": 6159.4407, "rew_std": 1555.6679, "Agent": "REINFORCE"}, {"env_step": 8632320, "rew": 5990.1787, "rew_std": 1561.9069, "Agent": "REINFORCE"}, {"env_step": 8663040, "rew": 6602.3922, "rew_std": 1792.703, "Agent": "REINFORCE"}, {"env_step": 8693760, "rew": 6646.9446, "rew_std": 1388.1919, "Agent": "REINFORCE"}, {"env_step": 8724480, "rew": 6394.6836, "rew_std": 970.0849, "Agent": "REINFORCE"}, {"env_step": 8755200, "rew": 6877.8872, "rew_std": 1009.9845, "Agent": "REINFORCE"}, {"env_step": 8785920, "rew": 6184.5929, "rew_std": 1464.8502, "Agent": "REINFORCE"}, {"env_step": 8816640, "rew": 6062.8531, "rew_std": 1367.4635, "Agent": "REINFORCE"}, {"env_step": 8847360, "rew": 6737.3933, "rew_std": 1261.609, "Agent": "REINFORCE"}, {"env_step": 8878080, "rew": 6322.801, "rew_std": 1068.3317, "Agent": "REINFORCE"}, {"env_step": 8908800, "rew": 6125.2032, "rew_std": 1196.5361, "Agent": "REINFORCE"}, {"env_step": 8939520, "rew": 6146.5552, "rew_std": 1153.7217, "Agent": "REINFORCE"}, {"env_step": 8970240, "rew": 6812.214, "rew_std": 530.805, "Agent": "REINFORCE"}, {"env_step": 9000960, "rew": 6844.0914, "rew_std": 841.6548, "Agent": "REINFORCE"}, {"env_step": 9031680, "rew": 6873.7066, "rew_std": 1135.0747, "Agent": "REINFORCE"}, {"env_step": 9062400, "rew": 6635.5172, "rew_std": 1004.5649, "Agent": "REINFORCE"}, {"env_step": 9093120, "rew": 6693.6973, "rew_std": 1015.4753, "Agent": "REINFORCE"}, {"env_step": 9123840, "rew": 6887.9459, "rew_std": 1380.6248, "Agent": "REINFORCE"}, {"env_step": 9154560, "rew": 6694.8057, "rew_std": 1666.3847, "Agent": "REINFORCE"}, {"env_step": 9185280, "rew": 6762.1576, "rew_std": 1212.6116, "Agent": "REINFORCE"}, {"env_step": 9216000, "rew": 7726.2164, "rew_std": 1287.2782, "Agent": "REINFORCE"}, {"env_step": 9246720, "rew": 6567.3651, "rew_std": 969.8238, "Agent": "REINFORCE"}, {"env_step": 9277440, "rew": 6641.0387, "rew_std": 1302.7611, "Agent": "REINFORCE"}, {"env_step": 9308160, "rew": 6559.1865, "rew_std": 1104.5484, "Agent": "REINFORCE"}, {"env_step": 9338880, "rew": 6207.0497, "rew_std": 1099.3404, "Agent": "REINFORCE"}, {"env_step": 9369600, "rew": 6919.7114, "rew_std": 1198.3527, "Agent": "REINFORCE"}, {"env_step": 9400320, "rew": 6900.913, "rew_std": 1189.5731, "Agent": "REINFORCE"}, {"env_step": 9431040, "rew": 6333.3599, "rew_std": 1552.1746, "Agent": "REINFORCE"}, {"env_step": 9461760, "rew": 6961.7171, "rew_std": 1292.5547, "Agent": "REINFORCE"}, {"env_step": 9492480, "rew": 6839.8277, "rew_std": 928.9957, "Agent": "REINFORCE"}, {"env_step": 9523200, "rew": 6278.2224, "rew_std": 572.1307, "Agent": "REINFORCE"}, {"env_step": 9553920, "rew": 6150.1885, "rew_std": 1659.5018, "Agent": "REINFORCE"}, {"env_step": 9584640, "rew": 6263.9494, "rew_std": 778.401, "Agent": "REINFORCE"}, {"env_step": 9615360, "rew": 6700.659, "rew_std": 743.8524, "Agent": "REINFORCE"}, {"env_step": 9646080, "rew": 6855.1353, "rew_std": 1217.9605, "Agent": "REINFORCE"}, {"env_step": 9676800, "rew": 7503.265, "rew_std": 839.6838, "Agent": "REINFORCE"}, {"env_step": 9707520, "rew": 7234.1868, "rew_std": 1186.2327, "Agent": "REINFORCE"}, {"env_step": 9738240, "rew": 7017.3017, "rew_std": 917.1484, "Agent": "REINFORCE"}, {"env_step": 9768960, "rew": 6725.5028, "rew_std": 800.6937, "Agent": "REINFORCE"}, {"env_step": 9799680, "rew": 7065.9637, "rew_std": 1153.5137, "Agent": "REINFORCE"}, {"env_step": 9830400, "rew": 6378.6522, "rew_std": 1429.952, "Agent": "REINFORCE"}, {"env_step": 9861120, "rew": 6962.9597, "rew_std": 1037.9087, "Agent": "REINFORCE"}, {"env_step": 9891840, "rew": 6958.6688, "rew_std": 1213.9064, "Agent": "REINFORCE"}, {"env_step": 9922560, "rew": 7098.3389, "rew_std": 1145.1282, "Agent": "REINFORCE"}, {"env_step": 9953280, "rew": 6609.1988, "rew_std": 1108.0279, "Agent": "REINFORCE"}, {"env_step": 9984000, "rew": 7164.6662, "rew_std": 1137.4225, "Agent": "REINFORCE"}, {"env_step": 10014720, "rew": 7364.7105, "rew_std": 993.5177, "Agent": "REINFORCE"}, {"env_step": 10045440, "rew": 6631.3363, "rew_std": 1164.1259, "Agent": "REINFORCE"}, {"env_step": 10076160, "rew": 6664.8013, "rew_std": 1606.3458, "Agent": "REINFORCE"}, {"env_step": 10106880, "rew": 6859.2966, "rew_std": 1411.2013, "Agent": "REINFORCE"}, {"env_step": 10137600, "rew": 6828.3037, "rew_std": 995.2747, "Agent": "REINFORCE"}, {"env_step": 10168320, "rew": 6260.87, "rew_std": 1026.5741, "Agent": "REINFORCE"}, {"env_step": 10199040, "rew": 6688.6403, "rew_std": 1283.3219, "Agent": "REINFORCE"}, {"env_step": 10229760, "rew": 7276.2345, "rew_std": 985.5105, "Agent": "REINFORCE"}, {"env_step": 0, "rew": 53.5626, "rew_std": 5.4712, "Agent": "A2C"}, {"env_step": 30000, "rew": 165.3699, "rew_std": 15.8262, "Agent": "A2C"}, {"env_step": 60000, "rew": 278.5414, "rew_std": 35.5195, "Agent": "A2C"}, {"env_step": 90000, "rew": 370.5591, "rew_std": 64.1232, "Agent": "A2C"}, {"env_step": 120000, "rew": 515.2061, "rew_std": 76.4682, "Agent": "A2C"}, {"env_step": 150000, "rew": 705.1444, "rew_std": 92.1359, "Agent": "A2C"}, {"env_step": 180000, "rew": 1125.0754, "rew_std": 350.0867, "Agent": "A2C"}, {"env_step": 210000, "rew": 3093.8255, "rew_std": 1674.9932, "Agent": "A2C"}, {"env_step": 240000, "rew": 6080.3757, "rew_std": 1434.3889, "Agent": "A2C"}, {"env_step": 270000, "rew": 6683.2966, "rew_std": 1128.3915, "Agent": "A2C"}, {"env_step": 300000, "rew": 7589.9427, "rew_std": 849.6975, "Agent": "A2C"}, {"env_step": 330000, "rew": 5696.9149, "rew_std": 2277.6442, "Agent": "A2C"}, {"env_step": 360000, "rew": 5649.5662, "rew_std": 1297.3898, "Agent": "A2C"}, {"env_step": 390000, "rew": 7372.0974, "rew_std": 2547.9211, "Agent": "A2C"}, {"env_step": 420000, "rew": 6570.9616, "rew_std": 1163.1758, "Agent": "A2C"}, {"env_step": 450000, "rew": 6874.2586, "rew_std": 1602.3641, "Agent": "A2C"}, {"env_step": 480000, "rew": 7311.5666, "rew_std": 999.7631, "Agent": "A2C"}, {"env_step": 510000, "rew": 7592.0021, "rew_std": 1130.0516, "Agent": "A2C"}, {"env_step": 540000, "rew": 7053.2387, "rew_std": 1230.277, "Agent": "A2C"}, {"env_step": 570000, "rew": 8249.9909, "rew_std": 1725.9754, "Agent": "A2C"}, {"env_step": 600000, "rew": 6397.1525, "rew_std": 1793.332, "Agent": "A2C"}, {"env_step": 630000, "rew": 7227.102, "rew_std": 1103.6105, "Agent": "A2C"}, {"env_step": 660000, "rew": 7133.5374, "rew_std": 1445.3755, "Agent": "A2C"}, {"env_step": 690000, "rew": 7732.3018, "rew_std": 1844.5892, "Agent": "A2C"}, {"env_step": 720000, "rew": 8703.7419, "rew_std": 834.3039, "Agent": "A2C"}, {"env_step": 750000, "rew": 8633.7331, "rew_std": 1107.9686, "Agent": "A2C"}, {"env_step": 780000, "rew": 8274.7136, "rew_std": 2053.5267, "Agent": "A2C"}, {"env_step": 810000, "rew": 7322.6685, "rew_std": 1656.3385, "Agent": "A2C"}, {"env_step": 840000, "rew": 7923.2505, "rew_std": 2314.1175, "Agent": "A2C"}, {"env_step": 870000, "rew": 7855.25, "rew_std": 2656.8457, "Agent": "A2C"}, {"env_step": 900000, "rew": 8149.0521, "rew_std": 1381.1895, "Agent": "A2C"}, {"env_step": 930000, "rew": 7485.3729, "rew_std": 3531.042, "Agent": "A2C"}, {"env_step": 960000, "rew": 9257.6636, "rew_std": 277.4383, "Agent": "A2C"}, {"env_step": 990000, "rew": 8989.5551, "rew_std": 604.5116, "Agent": "A2C"}, {"env_step": 1020000, "rew": 8710.0884, "rew_std": 717.8418, "Agent": "A2C"}, {"env_step": 1050000, "rew": 7964.6811, "rew_std": 2670.7761, "Agent": "A2C"}, {"env_step": 1080000, "rew": 8139.1856, "rew_std": 2183.9688, "Agent": "A2C"}, {"env_step": 1110000, "rew": 7763.4722, "rew_std": 2407.9368, "Agent": "A2C"}, {"env_step": 1140000, "rew": 7343.0162, "rew_std": 3320.6158, "Agent": "A2C"}, {"env_step": 1170000, "rew": 7926.8729, "rew_std": 2558.491, "Agent": "A2C"}, {"env_step": 1200000, "rew": 9074.6163, "rew_std": 591.3742, "Agent": "A2C"}, {"env_step": 1230000, "rew": 8704.4275, "rew_std": 832.6564, "Agent": "A2C"}, {"env_step": 1260000, "rew": 8379.228, "rew_std": 2616.9612, "Agent": "A2C"}, {"env_step": 1290000, "rew": 7995.9326, "rew_std": 2099.1052, "Agent": "A2C"}, {"env_step": 1320000, "rew": 8597.2945, "rew_std": 934.2636, "Agent": "A2C"}, {"env_step": 1350000, "rew": 9043.1429, "rew_std": 677.0861, "Agent": "A2C"}, {"env_step": 1380000, "rew": 9350.4301, "rew_std": 12.0039, "Agent": "A2C"}, {"env_step": 1410000, "rew": 8298.0241, "rew_std": 1903.4005, "Agent": "A2C"}, {"env_step": 1440000, "rew": 8109.2874, "rew_std": 2016.9284, "Agent": "A2C"}, {"env_step": 1470000, "rew": 8642.3605, "rew_std": 1534.7333, "Agent": "A2C"}, {"env_step": 1500000, "rew": 8981.4328, "rew_std": 607.3778, "Agent": "A2C"}, {"env_step": 1530000, "rew": 7579.4363, "rew_std": 3559.5222, "Agent": "A2C"}, {"env_step": 1560000, "rew": 6014.9854, "rew_std": 3158.7687, "Agent": "A2C"}, {"env_step": 1590000, "rew": 8436.0225, "rew_std": 911.7899, "Agent": "A2C"}, {"env_step": 1620000, "rew": 7240.4051, "rew_std": 2977.2566, "Agent": "A2C"}, {"env_step": 1650000, "rew": 7376.5496, "rew_std": 3101.1354, "Agent": "A2C"}, {"env_step": 1680000, "rew": 6505.8345, "rew_std": 3945.7676, "Agent": "A2C"}, {"env_step": 1710000, "rew": 8214.8968, "rew_std": 2053.7879, "Agent": "A2C"}, {"env_step": 1740000, "rew": 8406.5618, "rew_std": 1766.4248, "Agent": "A2C"}, {"env_step": 1770000, "rew": 8609.5837, "rew_std": 1270.4979, "Agent": "A2C"}, {"env_step": 1800000, "rew": 9170.2991, "rew_std": 525.8975, "Agent": "A2C"}, {"env_step": 1830000, "rew": 9347.1827, "rew_std": 13.3392, "Agent": "A2C"}, {"env_step": 1860000, "rew": 8803.5903, "rew_std": 918.7437, "Agent": "A2C"}, {"env_step": 1890000, "rew": 9161.4353, "rew_std": 374.3905, "Agent": "A2C"}, {"env_step": 1920000, "rew": 9348.3455, "rew_std": 12.6689, "Agent": "A2C"}, {"env_step": 1950000, "rew": 9176.7473, "rew_std": 543.232, "Agent": "A2C"}, {"env_step": 1980000, "rew": 7453.3149, "rew_std": 3360.1904, "Agent": "A2C"}, {"env_step": 2010000, "rew": 9351.3157, "rew_std": 12.7785, "Agent": "A2C"}, {"env_step": 2040000, "rew": 8813.3769, "rew_std": 1075.499, "Agent": "A2C"}, {"env_step": 2070000, "rew": 9077.6345, "rew_std": 816.8358, "Agent": "A2C"}, {"env_step": 2100000, "rew": 8904.5094, "rew_std": 716.6057, "Agent": "A2C"}, {"env_step": 2130000, "rew": 8614.7209, "rew_std": 1345.6491, "Agent": "A2C"}, {"env_step": 2160000, "rew": 8171.675, "rew_std": 2683.0986, "Agent": "A2C"}, {"env_step": 2190000, "rew": 7869.7354, "rew_std": 2363.8685, "Agent": "A2C"}, {"env_step": 2220000, "rew": 8819.4359, "rew_std": 596.9582, "Agent": "A2C"}, {"env_step": 2250000, "rew": 8387.8828, "rew_std": 2584.99, "Agent": "A2C"}, {"env_step": 2280000, "rew": 7908.1091, "rew_std": 1331.1717, "Agent": "A2C"}, {"env_step": 2310000, "rew": 8712.9974, "rew_std": 718.7488, "Agent": "A2C"}, {"env_step": 2340000, "rew": 8893.5525, "rew_std": 621.7979, "Agent": "A2C"}, {"env_step": 2370000, "rew": 7149.2746, "rew_std": 3312.735, "Agent": "A2C"}, {"env_step": 2400000, "rew": 8279.6943, "rew_std": 2617.049, "Agent": "A2C"}, {"env_step": 2430000, "rew": 9347.9399, "rew_std": 12.6763, "Agent": "A2C"}, {"env_step": 2460000, "rew": 8618.9958, "rew_std": 1134.2173, "Agent": "A2C"}, {"env_step": 2490000, "rew": 9167.1785, "rew_std": 555.3044, "Agent": "A2C"}, {"env_step": 2520000, "rew": 9112.2709, "rew_std": 722.9989, "Agent": "A2C"}, {"env_step": 2550000, "rew": 9346.3538, "rew_std": 14.2909, "Agent": "A2C"}, {"env_step": 2580000, "rew": 9261.4516, "rew_std": 260.8552, "Agent": "A2C"}, {"env_step": 2610000, "rew": 8807.666, "rew_std": 819.9374, "Agent": "A2C"}, {"env_step": 2640000, "rew": 8299.3944, "rew_std": 2555.8638, "Agent": "A2C"}, {"env_step": 2670000, "rew": 7549.846, "rew_std": 3158.768, "Agent": "A2C"}, {"env_step": 2700000, "rew": 8893.9045, "rew_std": 608.1252, "Agent": "A2C"}, {"env_step": 2730000, "rew": 8581.0372, "rew_std": 942.1541, "Agent": "A2C"}, {"env_step": 2760000, "rew": 8160.1186, "rew_std": 2406.8291, "Agent": "A2C"}, {"env_step": 2790000, "rew": 8313.3783, "rew_std": 2274.0883, "Agent": "A2C"}, {"env_step": 2820000, "rew": 8888.0056, "rew_std": 619.69, "Agent": "A2C"}, {"env_step": 2850000, "rew": 8804.6519, "rew_std": 1080.8811, "Agent": "A2C"}, {"env_step": 2880000, "rew": 9142.4464, "rew_std": 356.4098, "Agent": "A2C"}, {"env_step": 2910000, "rew": 8983.9959, "rew_std": 601.4488, "Agent": "A2C"}, {"env_step": 2940000, "rew": 9080.3266, "rew_std": 407.971, "Agent": "A2C"}, {"env_step": 2970000, "rew": 9349.8121, "rew_std": 7.0154, "Agent": "A2C"}, {"env_step": 3000000, "rew": 9081.4192, "rew_std": 570.8176, "Agent": "A2C"}] \ No newline at end of file diff --git a/examples/mujoco/benchmark/InvertedPendulum-v2/result.json b/examples/mujoco/benchmark/InvertedPendulum-v2/result.json new file mode 100644 index 0000000..0aaac28 --- /dev/null +++ b/examples/mujoco/benchmark/InvertedPendulum-v2/result.json @@ -0,0 +1 @@ +[{"env_step": 0, "rew": 10.32, "rew_std": 2.1711, "Agent": "SAC"}, {"env_step": 5000, "rew": 97.9, "rew_std": 54.3586, "Agent": "SAC"}, {"env_step": 10000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 15000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 20000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 25000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 30000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 35000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 40000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 45000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 50000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 55000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 60000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 65000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 70000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 75000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 80000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 85000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 90000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 95000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 100000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 105000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 110000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 115000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 120000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 125000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 130000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 135000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 140000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 145000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 150000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 155000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 160000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 165000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 170000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 175000, "rew": 920.69, "rew_std": 237.93, "Agent": "SAC"}, {"env_step": 180000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 185000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 190000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 195000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 200000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 205000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 210000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 215000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 220000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 225000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 230000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 235000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 240000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 245000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 250000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 255000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 260000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 265000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 270000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 275000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 280000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 285000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 290000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 295000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 300000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 305000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 310000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 315000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 320000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 325000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 330000, "rew": 994.56, "rew_std": 16.32, "Agent": "SAC"}, {"env_step": 335000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 340000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 345000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 350000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 355000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 360000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 365000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 370000, "rew": 988.24, "rew_std": 35.28, "Agent": "SAC"}, {"env_step": 375000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 380000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 385000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 390000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 395000, "rew": 912.58, "rew_std": 262.26, "Agent": "SAC"}, {"env_step": 400000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 405000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 410000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 415000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 420000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 425000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 430000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 435000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 440000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 445000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 450000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 455000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 460000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 465000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 470000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 475000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 480000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 485000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 490000, "rew": 924.73, "rew_std": 225.81, "Agent": "SAC"}, {"env_step": 495000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 500000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 505000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 510000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 515000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 520000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 525000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 530000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 535000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 540000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 545000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 550000, "rew": 942.67, "rew_std": 171.99, "Agent": "SAC"}, {"env_step": 555000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 560000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 565000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 570000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 575000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 580000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 585000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 590000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 595000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 600000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 605000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 610000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 615000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 620000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 625000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 630000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 635000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 640000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 645000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 650000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 655000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 660000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 665000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 670000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 675000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 680000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 685000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 690000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 695000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 700000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 705000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 710000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 715000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 720000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 725000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 730000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 735000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 740000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 745000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 750000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 755000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 760000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 765000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 770000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 775000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 780000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 785000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 790000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 795000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 800000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 805000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 810000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 815000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 820000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 825000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 830000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 835000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 840000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 845000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 850000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 855000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 860000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 865000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 870000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 875000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 880000, "rew": 957.33, "rew_std": 128.01, "Agent": "SAC"}, {"env_step": 885000, "rew": 907.27, "rew_std": 278.19, "Agent": "SAC"}, {"env_step": 890000, "rew": 944.61, "rew_std": 166.17, "Agent": "SAC"}, {"env_step": 895000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 900000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 905000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 910000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 915000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 920000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 925000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 930000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 935000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 940000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 945000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 950000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 955000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 960000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 965000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 970000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 975000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 980000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 985000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 990000, "rew": 971.67, "rew_std": 84.99, "Agent": "SAC"}, {"env_step": 995000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 1000000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 1005000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 1010000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 1015000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 1020000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 1025000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 1030000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 1035000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 1040000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 1045000, "rew": 907.31, "rew_std": 278.07, "Agent": "SAC"}, {"env_step": 1050000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 1055000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 1060000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 1065000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 1070000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 1075000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 1080000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 1085000, "rew": 909.78, "rew_std": 270.66, "Agent": "SAC"}, {"env_step": 1090000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 1095000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 1100000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 1105000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 1110000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 1115000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 1120000, "rew": 912.78, "rew_std": 261.66, "Agent": "SAC"}, {"env_step": 1125000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 1130000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 1135000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 1140000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 1145000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 1150000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 1155000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 1160000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 1165000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 1170000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 1175000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 1180000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 1185000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 1190000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 1195000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 1200000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 1205000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 1210000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 1215000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 1220000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 1225000, "rew": 906.82, "rew_std": 279.54, "Agent": "SAC"}, {"env_step": 1230000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 1235000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 1240000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 1245000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 1250000, "rew": 1000.0, "rew_std": 0.0, "Agent": "SAC"}, {"env_step": 0, "rew": 10.32, "rew_std": 2.1711, "Agent": "DDPG"}, {"env_step": 5000, "rew": 41.15, "rew_std": 17.2288, "Agent": "DDPG"}, {"env_step": 10000, "rew": 53.78, "rew_std": 29.9129, "Agent": "DDPG"}, {"env_step": 15000, "rew": 638.39, "rew_std": 444.4626, "Agent": "DDPG"}, {"env_step": 20000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 25000, "rew": 982.36, "rew_std": 52.92, "Agent": "DDPG"}, {"env_step": 30000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 35000, "rew": 946.08, "rew_std": 141.1384, "Agent": "DDPG"}, {"env_step": 40000, "rew": 922.44, "rew_std": 232.68, "Agent": "DDPG"}, {"env_step": 45000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 50000, "rew": 990.96, "rew_std": 27.12, "Agent": "DDPG"}, {"env_step": 55000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 60000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 65000, "rew": 904.88, "rew_std": 262.7022, "Agent": "DDPG"}, {"env_step": 70000, "rew": 996.64, "rew_std": 10.08, "Agent": "DDPG"}, {"env_step": 75000, "rew": 847.65, "rew_std": 310.1592, "Agent": "DDPG"}, {"env_step": 80000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 85000, "rew": 754.62, "rew_std": 378.6501, "Agent": "DDPG"}, {"env_step": 90000, "rew": 909.22, "rew_std": 272.34, "Agent": "DDPG"}, {"env_step": 95000, "rew": 906.05, "rew_std": 281.85, "Agent": "DDPG"}, {"env_step": 100000, "rew": 907.39, "rew_std": 277.83, "Agent": "DDPG"}, {"env_step": 105000, "rew": 959.77, "rew_std": 120.69, "Agent": "DDPG"}, {"env_step": 110000, "rew": 995.98, "rew_std": 12.06, "Agent": "DDPG"}, {"env_step": 115000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 120000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 125000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 130000, "rew": 909.5, "rew_std": 271.5, "Agent": "DDPG"}, {"env_step": 135000, "rew": 909.05, "rew_std": 272.85, "Agent": "DDPG"}, {"env_step": 140000, "rew": 980.01, "rew_std": 59.97, "Agent": "DDPG"}, {"env_step": 145000, "rew": 912.67, "rew_std": 261.99, "Agent": "DDPG"}, {"env_step": 150000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 155000, "rew": 844.36, "rew_std": 321.8701, "Agent": "DDPG"}, {"env_step": 160000, "rew": 964.87, "rew_std": 105.39, "Agent": "DDPG"}, {"env_step": 165000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 170000, "rew": 817.01, "rew_std": 366.5122, "Agent": "DDPG"}, {"env_step": 175000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 180000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 185000, "rew": 908.82, "rew_std": 252.7725, "Agent": "DDPG"}, {"env_step": 190000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 195000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 200000, "rew": 904.26, "rew_std": 287.22, "Agent": "DDPG"}, {"env_step": 205000, "rew": 862.49, "rew_std": 294.1062, "Agent": "DDPG"}, {"env_step": 210000, "rew": 990.86, "rew_std": 27.42, "Agent": "DDPG"}, {"env_step": 215000, "rew": 895.51, "rew_std": 257.5641, "Agent": "DDPG"}, {"env_step": 220000, "rew": 773.93, "rew_std": 349.0457, "Agent": "DDPG"}, {"env_step": 225000, "rew": 910.11, "rew_std": 269.67, "Agent": "DDPG"}, {"env_step": 230000, "rew": 867.81, "rew_std": 292.9672, "Agent": "DDPG"}, {"env_step": 235000, "rew": 818.76, "rew_std": 325.96, "Agent": "DDPG"}, {"env_step": 240000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 245000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 250000, "rew": 968.3, "rew_std": 71.1283, "Agent": "DDPG"}, {"env_step": 255000, "rew": 812.26, "rew_std": 375.4814, "Agent": "DDPG"}, {"env_step": 260000, "rew": 990.07, "rew_std": 29.79, "Agent": "DDPG"}, {"env_step": 265000, "rew": 982.53, "rew_std": 52.41, "Agent": "DDPG"}, {"env_step": 270000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 275000, "rew": 869.48, "rew_std": 289.0383, "Agent": "DDPG"}, {"env_step": 280000, "rew": 921.04, "rew_std": 236.88, "Agent": "DDPG"}, {"env_step": 285000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 290000, "rew": 962.56, "rew_std": 112.32, "Agent": "DDPG"}, {"env_step": 295000, "rew": 686.91, "rew_std": 392.568, "Agent": "DDPG"}, {"env_step": 300000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 305000, "rew": 908.78, "rew_std": 273.66, "Agent": "DDPG"}, {"env_step": 310000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 315000, "rew": 911.93, "rew_std": 264.21, "Agent": "DDPG"}, {"env_step": 320000, "rew": 818.29, "rew_std": 341.6617, "Agent": "DDPG"}, {"env_step": 325000, "rew": 914.95, "rew_std": 229.7549, "Agent": "DDPG"}, {"env_step": 330000, "rew": 982.79, "rew_std": 51.63, "Agent": "DDPG"}, {"env_step": 335000, "rew": 906.27, "rew_std": 281.19, "Agent": "DDPG"}, {"env_step": 340000, "rew": 912.32, "rew_std": 263.04, "Agent": "DDPG"}, {"env_step": 345000, "rew": 995.31, "rew_std": 14.07, "Agent": "DDPG"}, {"env_step": 350000, "rew": 914.14, "rew_std": 257.58, "Agent": "DDPG"}, {"env_step": 355000, "rew": 912.27, "rew_std": 263.19, "Agent": "DDPG"}, {"env_step": 360000, "rew": 753.66, "rew_std": 377.72, "Agent": "DDPG"}, {"env_step": 365000, "rew": 923.03, "rew_std": 223.965, "Agent": "DDPG"}, {"env_step": 370000, "rew": 903.03, "rew_std": 290.91, "Agent": "DDPG"}, {"env_step": 375000, "rew": 828.6, "rew_std": 344.9558, "Agent": "DDPG"}, {"env_step": 380000, "rew": 909.52, "rew_std": 271.44, "Agent": "DDPG"}, {"env_step": 385000, "rew": 951.22, "rew_std": 146.34, "Agent": "DDPG"}, {"env_step": 390000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 395000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 400000, "rew": 964.94, "rew_std": 105.18, "Agent": "DDPG"}, {"env_step": 405000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 410000, "rew": 932.17, "rew_std": 203.49, "Agent": "DDPG"}, {"env_step": 415000, "rew": 981.03, "rew_std": 56.91, "Agent": "DDPG"}, {"env_step": 420000, "rew": 820.77, "rew_std": 337.9462, "Agent": "DDPG"}, {"env_step": 425000, "rew": 750.05, "rew_std": 388.838, "Agent": "DDPG"}, {"env_step": 430000, "rew": 990.12, "rew_std": 29.64, "Agent": "DDPG"}, {"env_step": 435000, "rew": 794.77, "rew_std": 353.8976, "Agent": "DDPG"}, {"env_step": 440000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 445000, "rew": 907.03, "rew_std": 278.91, "Agent": "DDPG"}, {"env_step": 450000, "rew": 980.83, "rew_std": 38.3643, "Agent": "DDPG"}, {"env_step": 455000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 460000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 465000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 470000, "rew": 864.28, "rew_std": 287.4623, "Agent": "DDPG"}, {"env_step": 475000, "rew": 998.64, "rew_std": 4.08, "Agent": "DDPG"}, {"env_step": 480000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 485000, "rew": 917.69, "rew_std": 246.93, "Agent": "DDPG"}, {"env_step": 490000, "rew": 994.05, "rew_std": 17.85, "Agent": "DDPG"}, {"env_step": 495000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 500000, "rew": 883.9, "rew_std": 193.0312, "Agent": "DDPG"}, {"env_step": 505000, "rew": 809.55, "rew_std": 381.3104, "Agent": "DDPG"}, {"env_step": 510000, "rew": 895.14, "rew_std": 234.18, "Agent": "DDPG"}, {"env_step": 515000, "rew": 919.31, "rew_std": 242.07, "Agent": "DDPG"}, {"env_step": 520000, "rew": 821.45, "rew_std": 357.8348, "Agent": "DDPG"}, {"env_step": 525000, "rew": 883.2, "rew_std": 291.5019, "Agent": "DDPG"}, {"env_step": 530000, "rew": 882.44, "rew_std": 235.4276, "Agent": "DDPG"}, {"env_step": 535000, "rew": 925.62, "rew_std": 223.14, "Agent": "DDPG"}, {"env_step": 540000, "rew": 962.11, "rew_std": 113.67, "Agent": "DDPG"}, {"env_step": 545000, "rew": 916.88, "rew_std": 249.36, "Agent": "DDPG"}, {"env_step": 550000, "rew": 968.45, "rew_std": 94.65, "Agent": "DDPG"}, {"env_step": 555000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 560000, "rew": 936.39, "rew_std": 138.9512, "Agent": "DDPG"}, {"env_step": 565000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 570000, "rew": 901.64, "rew_std": 285.901, "Agent": "DDPG"}, {"env_step": 575000, "rew": 860.19, "rew_std": 289.7419, "Agent": "DDPG"}, {"env_step": 580000, "rew": 794.28, "rew_std": 342.0737, "Agent": "DDPG"}, {"env_step": 585000, "rew": 908.6, "rew_std": 274.2, "Agent": "DDPG"}, {"env_step": 590000, "rew": 980.7, "rew_std": 57.9, "Agent": "DDPG"}, {"env_step": 595000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 600000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 605000, "rew": 944.19, "rew_std": 167.43, "Agent": "DDPG"}, {"env_step": 610000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 615000, "rew": 822.83, "rew_std": 338.3354, "Agent": "DDPG"}, {"env_step": 620000, "rew": 996.75, "rew_std": 9.75, "Agent": "DDPG"}, {"env_step": 625000, "rew": 876.11, "rew_std": 290.0732, "Agent": "DDPG"}, {"env_step": 630000, "rew": 829.63, "rew_std": 344.1681, "Agent": "DDPG"}, {"env_step": 635000, "rew": 851.7, "rew_std": 307.4943, "Agent": "DDPG"}, {"env_step": 640000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 645000, "rew": 914.73, "rew_std": 227.7837, "Agent": "DDPG"}, {"env_step": 650000, "rew": 780.85, "rew_std": 353.241, "Agent": "DDPG"}, {"env_step": 655000, "rew": 828.64, "rew_std": 346.8068, "Agent": "DDPG"}, {"env_step": 660000, "rew": 920.6, "rew_std": 238.2, "Agent": "DDPG"}, {"env_step": 665000, "rew": 695.43, "rew_std": 360.9379, "Agent": "DDPG"}, {"env_step": 670000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 675000, "rew": 889.81, "rew_std": 275.4378, "Agent": "DDPG"}, {"env_step": 680000, "rew": 921.47, "rew_std": 235.59, "Agent": "DDPG"}, {"env_step": 685000, "rew": 914.11, "rew_std": 257.67, "Agent": "DDPG"}, {"env_step": 690000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 695000, "rew": 912.09, "rew_std": 263.73, "Agent": "DDPG"}, {"env_step": 700000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 705000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 710000, "rew": 992.54, "rew_std": 22.38, "Agent": "DDPG"}, {"env_step": 715000, "rew": 908.83, "rew_std": 273.51, "Agent": "DDPG"}, {"env_step": 720000, "rew": 971.07, "rew_std": 86.79, "Agent": "DDPG"}, {"env_step": 725000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 730000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 735000, "rew": 989.89, "rew_std": 30.33, "Agent": "DDPG"}, {"env_step": 740000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 745000, "rew": 978.06, "rew_std": 44.4382, "Agent": "DDPG"}, {"env_step": 750000, "rew": 934.12, "rew_std": 197.64, "Agent": "DDPG"}, {"env_step": 755000, "rew": 918.17, "rew_std": 245.49, "Agent": "DDPG"}, {"env_step": 760000, "rew": 982.55, "rew_std": 52.35, "Agent": "DDPG"}, {"env_step": 765000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 770000, "rew": 906.26, "rew_std": 281.22, "Agent": "DDPG"}, {"env_step": 775000, "rew": 907.98, "rew_std": 276.06, "Agent": "DDPG"}, {"env_step": 780000, "rew": 849.33, "rew_std": 303.7142, "Agent": "DDPG"}, {"env_step": 785000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 790000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 795000, "rew": 963.47, "rew_std": 109.59, "Agent": "DDPG"}, {"env_step": 800000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 805000, "rew": 885.71, "rew_std": 283.8073, "Agent": "DDPG"}, {"env_step": 810000, "rew": 909.16, "rew_std": 272.52, "Agent": "DDPG"}, {"env_step": 815000, "rew": 887.46, "rew_std": 232.1516, "Agent": "DDPG"}, {"env_step": 820000, "rew": 990.9, "rew_std": 27.3, "Agent": "DDPG"}, {"env_step": 825000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 830000, "rew": 915.07, "rew_std": 254.79, "Agent": "DDPG"}, {"env_step": 835000, "rew": 992.61, "rew_std": 22.17, "Agent": "DDPG"}, {"env_step": 840000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 845000, "rew": 900.96, "rew_std": 297.12, "Agent": "DDPG"}, {"env_step": 850000, "rew": 924.3, "rew_std": 227.1, "Agent": "DDPG"}, {"env_step": 855000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 860000, "rew": 910.32, "rew_std": 269.04, "Agent": "DDPG"}, {"env_step": 865000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 870000, "rew": 972.08, "rew_std": 83.76, "Agent": "DDPG"}, {"env_step": 875000, "rew": 910.32, "rew_std": 269.04, "Agent": "DDPG"}, {"env_step": 880000, "rew": 907.29, "rew_std": 278.13, "Agent": "DDPG"}, {"env_step": 885000, "rew": 890.26, "rew_std": 269.6544, "Agent": "DDPG"}, {"env_step": 890000, "rew": 951.63, "rew_std": 120.6337, "Agent": "DDPG"}, {"env_step": 895000, "rew": 929.04, "rew_std": 212.88, "Agent": "DDPG"}, {"env_step": 900000, "rew": 925.66, "rew_std": 193.9828, "Agent": "DDPG"}, {"env_step": 905000, "rew": 914.34, "rew_std": 256.98, "Agent": "DDPG"}, {"env_step": 910000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 915000, "rew": 973.56, "rew_std": 79.32, "Agent": "DDPG"}, {"env_step": 920000, "rew": 834.16, "rew_std": 333.6571, "Agent": "DDPG"}, {"env_step": 925000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 930000, "rew": 818.05, "rew_std": 364.085, "Agent": "DDPG"}, {"env_step": 935000, "rew": 847.89, "rew_std": 308.9834, "Agent": "DDPG"}, {"env_step": 940000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 945000, "rew": 963.11, "rew_std": 110.67, "Agent": "DDPG"}, {"env_step": 950000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 955000, "rew": 905.48, "rew_std": 283.56, "Agent": "DDPG"}, {"env_step": 960000, "rew": 942.27, "rew_std": 146.4769, "Agent": "DDPG"}, {"env_step": 965000, "rew": 853.11, "rew_std": 275.556, "Agent": "DDPG"}, {"env_step": 970000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 975000, "rew": 912.17, "rew_std": 263.49, "Agent": "DDPG"}, {"env_step": 980000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 985000, "rew": 923.86, "rew_std": 228.42, "Agent": "DDPG"}, {"env_step": 990000, "rew": 854.05, "rew_std": 314.9146, "Agent": "DDPG"}, {"env_step": 995000, "rew": 736.91, "rew_std": 403.1681, "Agent": "DDPG"}, {"env_step": 1000000, "rew": 924.53, "rew_std": 226.41, "Agent": "DDPG"}, {"env_step": 1005000, "rew": 996.16, "rew_std": 11.52, "Agent": "DDPG"}, {"env_step": 1010000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 1015000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 1020000, "rew": 840.45, "rew_std": 257.3666, "Agent": "DDPG"}, {"env_step": 1025000, "rew": 822.64, "rew_std": 354.8743, "Agent": "DDPG"}, {"env_step": 1030000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 1035000, "rew": 910.85, "rew_std": 267.45, "Agent": "DDPG"}, {"env_step": 1040000, "rew": 912.68, "rew_std": 261.96, "Agent": "DDPG"}, {"env_step": 1045000, "rew": 945.21, "rew_std": 164.37, "Agent": "DDPG"}, {"env_step": 1050000, "rew": 980.2, "rew_std": 59.4, "Agent": "DDPG"}, {"env_step": 1055000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 1060000, "rew": 816.61, "rew_std": 366.8583, "Agent": "DDPG"}, {"env_step": 1065000, "rew": 908.51, "rew_std": 274.47, "Agent": "DDPG"}, {"env_step": 1070000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 1075000, "rew": 907.18, "rew_std": 278.46, "Agent": "DDPG"}, {"env_step": 1080000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 1085000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 1090000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 1095000, "rew": 980.25, "rew_std": 59.25, "Agent": "DDPG"}, {"env_step": 1100000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 1105000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 1110000, "rew": 910.12, "rew_std": 269.64, "Agent": "DDPG"}, {"env_step": 1115000, "rew": 766.1, "rew_std": 364.7707, "Agent": "DDPG"}, {"env_step": 1120000, "rew": 908.51, "rew_std": 264.2015, "Agent": "DDPG"}, {"env_step": 1125000, "rew": 958.67, "rew_std": 123.99, "Agent": "DDPG"}, {"env_step": 1130000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 1135000, "rew": 994.92, "rew_std": 15.24, "Agent": "DDPG"}, {"env_step": 1140000, "rew": 971.41, "rew_std": 85.77, "Agent": "DDPG"}, {"env_step": 1145000, "rew": 925.66, "rew_std": 223.02, "Agent": "DDPG"}, {"env_step": 1150000, "rew": 907.61, "rew_std": 277.17, "Agent": "DDPG"}, {"env_step": 1155000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 1160000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 1165000, "rew": 851.25, "rew_std": 299.4327, "Agent": "DDPG"}, {"env_step": 1170000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 1175000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 1180000, "rew": 994.58, "rew_std": 16.26, "Agent": "DDPG"}, {"env_step": 1185000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 1190000, "rew": 990.83, "rew_std": 27.51, "Agent": "DDPG"}, {"env_step": 1195000, "rew": 911.21, "rew_std": 266.37, "Agent": "DDPG"}, {"env_step": 1200000, "rew": 905.24, "rew_std": 284.28, "Agent": "DDPG"}, {"env_step": 1205000, "rew": 943.31, "rew_std": 170.07, "Agent": "DDPG"}, {"env_step": 1210000, "rew": 990.66, "rew_std": 28.02, "Agent": "DDPG"}, {"env_step": 1215000, "rew": 954.84, "rew_std": 135.48, "Agent": "DDPG"}, {"env_step": 1220000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 1225000, "rew": 990.61, "rew_std": 28.17, "Agent": "DDPG"}, {"env_step": 1230000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 1235000, "rew": 962.42, "rew_std": 112.74, "Agent": "DDPG"}, {"env_step": 1240000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 1245000, "rew": 802.56, "rew_std": 334.6394, "Agent": "DDPG"}, {"env_step": 1250000, "rew": 1000.0, "rew_std": 0.0, "Agent": "DDPG"}, {"env_step": 5000, "rew": 57.22, "rew_std": 8.7888, "Agent": "TD3"}, {"env_step": 10000, "rew": 63.94, "rew_std": 15.7719, "Agent": "TD3"}, {"env_step": 15000, "rew": 58.57, "rew_std": 17.0332, "Agent": "TD3"}, {"env_step": 20000, "rew": 126.66, "rew_std": 76.2021, "Agent": "TD3"}, {"env_step": 25000, "rew": 616.43, "rew_std": 388.5243, "Agent": "TD3"}, {"env_step": 30000, "rew": 828.24, "rew_std": 343.5527, "Agent": "TD3"}, {"env_step": 35000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 40000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 45000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 50000, "rew": 933.47, "rew_std": 199.59, "Agent": "TD3"}, {"env_step": 55000, "rew": 976.87, "rew_std": 69.39, "Agent": "TD3"}, {"env_step": 60000, "rew": 809.04, "rew_std": 381.9246, "Agent": "TD3"}, {"env_step": 65000, "rew": 808.35, "rew_std": 383.3043, "Agent": "TD3"}, {"env_step": 70000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 75000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 80000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 85000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 90000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 95000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 100000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 105000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 110000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 115000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 120000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 125000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 130000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 135000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 140000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 145000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 150000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 155000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 160000, "rew": 903.53, "rew_std": 289.41, "Agent": "TD3"}, {"env_step": 165000, "rew": 904.12, "rew_std": 287.64, "Agent": "TD3"}, {"env_step": 170000, "rew": 904.97, "rew_std": 285.09, "Agent": "TD3"}, {"env_step": 175000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 180000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 185000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 190000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 195000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 200000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 205000, "rew": 903.88, "rew_std": 288.36, "Agent": "TD3"}, {"env_step": 210000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 215000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 220000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 225000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 230000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 235000, "rew": 981.89, "rew_std": 54.33, "Agent": "TD3"}, {"env_step": 240000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 245000, "rew": 955.73, "rew_std": 132.81, "Agent": "TD3"}, {"env_step": 250000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 255000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 260000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 265000, "rew": 919.42, "rew_std": 241.74, "Agent": "TD3"}, {"env_step": 270000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 275000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 280000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 285000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 290000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 295000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 300000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 305000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 310000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 315000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 320000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 325000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 330000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 335000, "rew": 995.98, "rew_std": 12.06, "Agent": "TD3"}, {"env_step": 340000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 345000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 350000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 355000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 360000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 365000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 370000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 375000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 380000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 385000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 390000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 395000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 400000, "rew": 987.18, "rew_std": 38.46, "Agent": "TD3"}, {"env_step": 405000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 410000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 415000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 420000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 425000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 430000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 435000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 440000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 445000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 450000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 455000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 460000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 465000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 470000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 475000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 480000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 485000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 490000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 495000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 500000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 505000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 510000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 515000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 520000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 525000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 530000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 535000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 540000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 545000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 550000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 555000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 560000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 565000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 570000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 575000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 580000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 585000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 590000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 595000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 600000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 605000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 610000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 615000, "rew": 912.84, "rew_std": 261.48, "Agent": "TD3"}, {"env_step": 620000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 625000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 630000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 635000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 640000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 645000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 650000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 655000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 660000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 665000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 670000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 675000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 680000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 685000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 690000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 695000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 700000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 705000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 710000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 715000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 720000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 725000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 730000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 735000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 740000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 745000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 750000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 755000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 760000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 765000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 770000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 775000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 780000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 785000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 790000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 795000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 800000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 805000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 810000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 815000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 820000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 825000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 830000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 835000, "rew": 967.62, "rew_std": 97.14, "Agent": "TD3"}, {"env_step": 840000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 845000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 850000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 855000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 860000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 865000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 870000, "rew": 960.62, "rew_std": 118.14, "Agent": "TD3"}, {"env_step": 875000, "rew": 903.47, "rew_std": 289.59, "Agent": "TD3"}, {"env_step": 880000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 885000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 890000, "rew": 953.41, "rew_std": 139.77, "Agent": "TD3"}, {"env_step": 895000, "rew": 916.7, "rew_std": 249.9, "Agent": "TD3"}, {"env_step": 900000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 905000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 910000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 915000, "rew": 992.77, "rew_std": 21.69, "Agent": "TD3"}, {"env_step": 920000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 925000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 930000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 935000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 940000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 945000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 950000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 955000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 960000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 965000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 970000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 975000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 980000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 985000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 990000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 995000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 1000000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 1005000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 1010000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 1015000, "rew": 927.09, "rew_std": 218.73, "Agent": "TD3"}, {"env_step": 1020000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 1025000, "rew": 930.37, "rew_std": 208.89, "Agent": "TD3"}, {"env_step": 1030000, "rew": 930.54, "rew_std": 208.38, "Agent": "TD3"}, {"env_step": 1035000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 1040000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 1045000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 1050000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 1055000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 1060000, "rew": 980.63, "rew_std": 58.11, "Agent": "TD3"}, {"env_step": 1065000, "rew": 971.03, "rew_std": 86.91, "Agent": "TD3"}, {"env_step": 1070000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 1075000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 1080000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 1085000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 1090000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 1095000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 1100000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 1105000, "rew": 966.52, "rew_std": 73.5236, "Agent": "TD3"}, {"env_step": 1110000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 1115000, "rew": 960.61, "rew_std": 118.17, "Agent": "TD3"}, {"env_step": 1120000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 1125000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 1130000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 1135000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 1140000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 1145000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 1150000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 1155000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 1160000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 1165000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 1170000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 1175000, "rew": 913.93, "rew_std": 258.21, "Agent": "TD3"}, {"env_step": 1180000, "rew": 991.61, "rew_std": 25.17, "Agent": "TD3"}, {"env_step": 1185000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 1190000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 1195000, "rew": 906.07, "rew_std": 281.79, "Agent": "TD3"}, {"env_step": 1200000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 1205000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 1210000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 1215000, "rew": 966.09, "rew_std": 101.73, "Agent": "TD3"}, {"env_step": 1220000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 1225000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 1230000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 1235000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 1240000, "rew": 1000.0, "rew_std": 0.0, "Agent": "TD3"}, {"env_step": 1245000, "rew": 909.42, "rew_std": 271.74, "Agent": "TD3"}, {"env_step": 1250000, "rew": 983.56, "rew_std": 49.32, "Agent": "TD3"}, {"env_step": 0, "rew": -46.1947, "rew_std": 25.5804, "Agent": "TRPO"}, {"env_step": 30720, "rew": -13.9012, "rew_std": 11.913, "Agent": "TRPO"}, {"env_step": 61440, "rew": 18.0615, "rew_std": 9.0166, "Agent": "TRPO"}, {"env_step": 92160, "rew": 51.9175, "rew_std": 16.4955, "Agent": "TRPO"}, {"env_step": 122880, "rew": 108.4958, "rew_std": 33.2241, "Agent": "TRPO"}, {"env_step": 153600, "rew": 183.9944, "rew_std": 35.8262, "Agent": "TRPO"}, {"env_step": 184320, "rew": 257.4861, "rew_std": 60.051, "Agent": "TRPO"}, {"env_step": 215040, "rew": 430.4068, "rew_std": 124.2541, "Agent": "TRPO"}, {"env_step": 245760, "rew": 564.2017, "rew_std": 146.0606, "Agent": "TRPO"}, {"env_step": 276480, "rew": 682.5168, "rew_std": 122.3187, "Agent": "TRPO"}, {"env_step": 307200, "rew": 832.5033, "rew_std": 159.2968, "Agent": "TRPO"}, {"env_step": 337920, "rew": 1014.1049, "rew_std": 221.8108, "Agent": "TRPO"}, {"env_step": 368640, "rew": 1128.5618, "rew_std": 167.9213, "Agent": "TRPO"}, {"env_step": 399360, "rew": 1146.4494, "rew_std": 215.1602, "Agent": "TRPO"}, {"env_step": 430080, "rew": 1233.6374, "rew_std": 286.8604, "Agent": "TRPO"}, {"env_step": 460800, "rew": 1261.4876, "rew_std": 301.0892, "Agent": "TRPO"}, {"env_step": 491520, "rew": 1472.6865, "rew_std": 215.7148, "Agent": "TRPO"}, {"env_step": 522240, "rew": 1501.5343, "rew_std": 428.1053, "Agent": "TRPO"}, {"env_step": 552960, "rew": 1631.5161, "rew_std": 558.1692, "Agent": "TRPO"}, {"env_step": 583680, "rew": 1741.6005, "rew_std": 483.7599, "Agent": "TRPO"}, {"env_step": 614400, "rew": 1809.3667, "rew_std": 444.8329, "Agent": "TRPO"}, {"env_step": 645120, "rew": 1943.2351, "rew_std": 386.0455, "Agent": "TRPO"}, {"env_step": 675840, "rew": 2119.3593, "rew_std": 463.9853, "Agent": "TRPO"}, {"env_step": 706560, "rew": 2160.9349, "rew_std": 299.8534, "Agent": "TRPO"}, {"env_step": 737280, "rew": 2205.6584, "rew_std": 495.8226, "Agent": "TRPO"}, {"env_step": 768000, "rew": 2148.4616, "rew_std": 617.4263, "Agent": "TRPO"}, {"env_step": 798720, "rew": 2294.2347, "rew_std": 619.654, "Agent": "TRPO"}, {"env_step": 829440, "rew": 2334.3877, "rew_std": 569.8366, "Agent": "TRPO"}, {"env_step": 860160, "rew": 2373.0195, "rew_std": 635.4872, "Agent": "TRPO"}, {"env_step": 890880, "rew": 2572.1874, "rew_std": 771.2781, "Agent": "TRPO"}, {"env_step": 921600, "rew": 2568.2581, "rew_std": 539.7726, "Agent": "TRPO"}, {"env_step": 952320, "rew": 2639.2867, "rew_std": 572.4301, "Agent": "TRPO"}, {"env_step": 983040, "rew": 2866.6508, "rew_std": 707.8937, "Agent": "TRPO"}, {"env_step": 1013760, "rew": 2731.0697, "rew_std": 675.644, "Agent": "TRPO"}, {"env_step": 1044480, "rew": 2702.1965, "rew_std": 742.4807, "Agent": "TRPO"}, {"env_step": 1075200, "rew": 3001.7621, "rew_std": 688.8314, "Agent": "TRPO"}, {"env_step": 1105920, "rew": 2877.8716, "rew_std": 922.2411, "Agent": "TRPO"}, {"env_step": 1136640, "rew": 3104.9072, "rew_std": 800.8227, "Agent": "TRPO"}, {"env_step": 1167360, "rew": 2976.5528, "rew_std": 661.5153, "Agent": "TRPO"}, {"env_step": 1198080, "rew": 3196.2228, "rew_std": 707.9679, "Agent": "TRPO"}, {"env_step": 1228800, "rew": 3237.2005, "rew_std": 505.6685, "Agent": "TRPO"}, {"env_step": 1259520, "rew": 3098.3083, "rew_std": 578.4346, "Agent": "TRPO"}, {"env_step": 1290240, "rew": 3356.962, "rew_std": 786.3838, "Agent": "TRPO"}, {"env_step": 1320960, "rew": 3241.2647, "rew_std": 600.3495, "Agent": "TRPO"}, {"env_step": 1351680, "rew": 3361.1722, "rew_std": 547.5151, "Agent": "TRPO"}, {"env_step": 1382400, "rew": 3495.3879, "rew_std": 662.5387, "Agent": "TRPO"}, {"env_step": 1413120, "rew": 3388.8443, "rew_std": 537.1016, "Agent": "TRPO"}, {"env_step": 1443840, "rew": 3493.6456, "rew_std": 509.2391, "Agent": "TRPO"}, {"env_step": 1474560, "rew": 3256.3625, "rew_std": 412.1903, "Agent": "TRPO"}, {"env_step": 1505280, "rew": 3695.3862, "rew_std": 644.5284, "Agent": "TRPO"}, {"env_step": 1536000, "rew": 3685.0253, "rew_std": 537.4812, "Agent": "TRPO"}, {"env_step": 1566720, "rew": 3565.3736, "rew_std": 647.2313, "Agent": "TRPO"}, {"env_step": 1597440, "rew": 3790.5153, "rew_std": 702.1943, "Agent": "TRPO"}, {"env_step": 1628160, "rew": 3598.3267, "rew_std": 626.2541, "Agent": "TRPO"}, {"env_step": 1658880, "rew": 3970.7287, "rew_std": 647.9836, "Agent": "TRPO"}, {"env_step": 1689600, "rew": 3658.6492, "rew_std": 645.0697, "Agent": "TRPO"}, {"env_step": 1720320, "rew": 3848.0345, "rew_std": 701.1273, "Agent": "TRPO"}, {"env_step": 1751040, "rew": 4061.4513, "rew_std": 690.0289, "Agent": "TRPO"}, {"env_step": 1781760, "rew": 3807.921, "rew_std": 706.5999, "Agent": "TRPO"}, {"env_step": 1812480, "rew": 3802.0342, "rew_std": 663.7139, "Agent": "TRPO"}, {"env_step": 1843200, "rew": 3873.6662, "rew_std": 566.3833, "Agent": "TRPO"}, {"env_step": 1873920, "rew": 4038.6082, "rew_std": 518.5079, "Agent": "TRPO"}, {"env_step": 1904640, "rew": 4167.3221, "rew_std": 440.9658, "Agent": "TRPO"}, {"env_step": 1935360, "rew": 3922.2855, "rew_std": 733.8238, "Agent": "TRPO"}, {"env_step": 1966080, "rew": 4174.7307, "rew_std": 495.1849, "Agent": "TRPO"}, {"env_step": 1996800, "rew": 4155.8384, "rew_std": 477.6376, "Agent": "TRPO"}, {"env_step": 2027520, "rew": 4259.3055, "rew_std": 577.3636, "Agent": "TRPO"}, {"env_step": 2058240, "rew": 4256.1933, "rew_std": 594.2826, "Agent": "TRPO"}, {"env_step": 2088960, "rew": 4355.0669, "rew_std": 605.8978, "Agent": "TRPO"}, {"env_step": 2119680, "rew": 4297.2252, "rew_std": 474.421, "Agent": "TRPO"}, {"env_step": 2150400, "rew": 4287.4698, "rew_std": 363.7332, "Agent": "TRPO"}, {"env_step": 2181120, "rew": 4398.2137, "rew_std": 354.3603, "Agent": "TRPO"}, {"env_step": 2211840, "rew": 4413.5796, "rew_std": 460.9681, "Agent": "TRPO"}, {"env_step": 2242560, "rew": 4329.001, "rew_std": 530.2755, "Agent": "TRPO"}, {"env_step": 2273280, "rew": 4252.3464, "rew_std": 597.5882, "Agent": "TRPO"}, {"env_step": 2304000, "rew": 4462.5657, "rew_std": 495.3802, "Agent": "TRPO"}, {"env_step": 2334720, "rew": 4374.2838, "rew_std": 457.0956, "Agent": "TRPO"}, {"env_step": 2365440, "rew": 4529.4549, "rew_std": 461.1733, "Agent": "TRPO"}, {"env_step": 2396160, "rew": 4291.4468, "rew_std": 411.2142, "Agent": "TRPO"}, {"env_step": 2426880, "rew": 4430.1459, "rew_std": 601.8812, "Agent": "TRPO"}, {"env_step": 2457600, "rew": 4508.078, "rew_std": 465.0507, "Agent": "TRPO"}, {"env_step": 2488320, "rew": 4151.3796, "rew_std": 414.656, "Agent": "TRPO"}, {"env_step": 2519040, "rew": 4705.7212, "rew_std": 477.1206, "Agent": "TRPO"}, {"env_step": 2549760, "rew": 4641.3111, "rew_std": 622.6369, "Agent": "TRPO"}, {"env_step": 2580480, "rew": 4484.1452, "rew_std": 442.0811, "Agent": "TRPO"}, {"env_step": 2611200, "rew": 4658.7294, "rew_std": 432.1496, "Agent": "TRPO"}, {"env_step": 2641920, "rew": 4625.8785, "rew_std": 357.4208, "Agent": "TRPO"}, {"env_step": 2672640, "rew": 4284.9794, "rew_std": 599.7262, "Agent": "TRPO"}, {"env_step": 2703360, "rew": 4419.1972, "rew_std": 447.5325, "Agent": "TRPO"}, {"env_step": 2734080, "rew": 4746.1067, "rew_std": 580.4515, "Agent": "TRPO"}, {"env_step": 2764800, "rew": 4756.6644, "rew_std": 502.4658, "Agent": "TRPO"}, {"env_step": 2795520, "rew": 4607.7287, "rew_std": 427.1984, "Agent": "TRPO"}, {"env_step": 2826240, "rew": 4404.4363, "rew_std": 555.3297, "Agent": "TRPO"}, {"env_step": 2856960, "rew": 4816.8623, "rew_std": 528.6794, "Agent": "TRPO"}, {"env_step": 2887680, "rew": 4753.3578, "rew_std": 715.7057, "Agent": "TRPO"}, {"env_step": 2918400, "rew": 4604.763, "rew_std": 447.4458, "Agent": "TRPO"}, {"env_step": 2949120, "rew": 4800.2693, "rew_std": 602.9667, "Agent": "TRPO"}, {"env_step": 2979840, "rew": 4772.9396, "rew_std": 368.1691, "Agent": "TRPO"}, {"env_step": 3010560, "rew": 4897.0257, "rew_std": 449.1329, "Agent": "TRPO"}, {"env_step": 3041280, "rew": 4841.7814, "rew_std": 534.8951, "Agent": "TRPO"}, {"env_step": 3072000, "rew": 4604.9043, "rew_std": 580.976, "Agent": "TRPO"}, {"env_step": 0, "rew": -46.1947, "rew_std": 25.5804, "Agent": "NPG"}, {"env_step": 30720, "rew": -29.0009, "rew_std": 15.8916, "Agent": "NPG"}, {"env_step": 61440, "rew": -10.4279, "rew_std": 11.0242, "Agent": "NPG"}, {"env_step": 92160, "rew": 11.1187, "rew_std": 9.0737, "Agent": "NPG"}, {"env_step": 122880, "rew": 27.5852, "rew_std": 16.4638, "Agent": "NPG"}, {"env_step": 153600, "rew": 56.7285, "rew_std": 23.8642, "Agent": "NPG"}, {"env_step": 184320, "rew": 81.2319, "rew_std": 29.1525, "Agent": "NPG"}, {"env_step": 215040, "rew": 159.4704, "rew_std": 47.3793, "Agent": "NPG"}, {"env_step": 245760, "rew": 174.9781, "rew_std": 57.693, "Agent": "NPG"}, {"env_step": 276480, "rew": 178.6358, "rew_std": 32.807, "Agent": "NPG"}, {"env_step": 307200, "rew": 218.3212, "rew_std": 41.035, "Agent": "NPG"}, {"env_step": 337920, "rew": 368.663, "rew_std": 80.0623, "Agent": "NPG"}, {"env_step": 368640, "rew": 432.2163, "rew_std": 63.1225, "Agent": "NPG"}, {"env_step": 399360, "rew": 568.0807, "rew_std": 125.517, "Agent": "NPG"}, {"env_step": 430080, "rew": 631.1259, "rew_std": 114.6371, "Agent": "NPG"}, {"env_step": 460800, "rew": 753.5929, "rew_std": 118.5185, "Agent": "NPG"}, {"env_step": 491520, "rew": 864.7371, "rew_std": 183.4071, "Agent": "NPG"}, {"env_step": 522240, "rew": 940.2978, "rew_std": 158.8018, "Agent": "NPG"}, {"env_step": 552960, "rew": 1021.7661, "rew_std": 159.2406, "Agent": "NPG"}, {"env_step": 583680, "rew": 1131.714, "rew_std": 150.4756, "Agent": "NPG"}, {"env_step": 614400, "rew": 1201.5181, "rew_std": 164.258, "Agent": "NPG"}, {"env_step": 645120, "rew": 1343.1191, "rew_std": 184.6686, "Agent": "NPG"}, {"env_step": 675840, "rew": 1318.401, "rew_std": 229.7468, "Agent": "NPG"}, {"env_step": 706560, "rew": 1459.1159, "rew_std": 222.6298, "Agent": "NPG"}, {"env_step": 737280, "rew": 1542.0295, "rew_std": 203.9479, "Agent": "NPG"}, {"env_step": 768000, "rew": 1634.9176, "rew_std": 218.2131, "Agent": "NPG"}, {"env_step": 798720, "rew": 1692.0983, "rew_std": 343.9785, "Agent": "NPG"}, {"env_step": 829440, "rew": 1906.1507, "rew_std": 457.5102, "Agent": "NPG"}, {"env_step": 860160, "rew": 1775.1748, "rew_std": 342.0144, "Agent": "NPG"}, {"env_step": 890880, "rew": 2102.0494, "rew_std": 355.8503, "Agent": "NPG"}, {"env_step": 921600, "rew": 2046.7794, "rew_std": 340.8884, "Agent": "NPG"}, {"env_step": 952320, "rew": 2164.3512, "rew_std": 405.7482, "Agent": "NPG"}, {"env_step": 983040, "rew": 2348.312, "rew_std": 509.9401, "Agent": "NPG"}, {"env_step": 1013760, "rew": 2365.8958, "rew_std": 523.6603, "Agent": "NPG"}, {"env_step": 1044480, "rew": 2324.9164, "rew_std": 528.4526, "Agent": "NPG"}, {"env_step": 1075200, "rew": 2493.722, "rew_std": 583.8895, "Agent": "NPG"}, {"env_step": 1105920, "rew": 2638.7282, "rew_std": 650.3356, "Agent": "NPG"}, {"env_step": 1136640, "rew": 2588.3598, "rew_std": 506.4239, "Agent": "NPG"}, {"env_step": 1167360, "rew": 2657.8218, "rew_std": 542.1361, "Agent": "NPG"}, {"env_step": 1198080, "rew": 2772.0887, "rew_std": 399.8722, "Agent": "NPG"}, {"env_step": 1228800, "rew": 2983.8532, "rew_std": 405.1427, "Agent": "NPG"}, {"env_step": 1259520, "rew": 2969.912, "rew_std": 543.0224, "Agent": "NPG"}, {"env_step": 1290240, "rew": 2898.5856, "rew_std": 687.1927, "Agent": "NPG"}, {"env_step": 1320960, "rew": 2979.687, "rew_std": 685.3181, "Agent": "NPG"}, {"env_step": 1351680, "rew": 2937.7865, "rew_std": 654.2811, "Agent": "NPG"}, {"env_step": 1382400, "rew": 3180.3262, "rew_std": 502.1675, "Agent": "NPG"}, {"env_step": 1413120, "rew": 3181.4746, "rew_std": 526.9918, "Agent": "NPG"}, {"env_step": 1443840, "rew": 3186.725, "rew_std": 483.0104, "Agent": "NPG"}, {"env_step": 1474560, "rew": 3347.1624, "rew_std": 474.3212, "Agent": "NPG"}, {"env_step": 1505280, "rew": 3264.1247, "rew_std": 542.1326, "Agent": "NPG"}, {"env_step": 1536000, "rew": 3267.3737, "rew_std": 536.477, "Agent": "NPG"}, {"env_step": 1566720, "rew": 3677.709, "rew_std": 486.0857, "Agent": "NPG"}, {"env_step": 1597440, "rew": 3379.1083, "rew_std": 683.1763, "Agent": "NPG"}, {"env_step": 1628160, "rew": 3533.6653, "rew_std": 437.9623, "Agent": "NPG"}, {"env_step": 1658880, "rew": 3534.0292, "rew_std": 787.6867, "Agent": "NPG"}, {"env_step": 1689600, "rew": 3399.7589, "rew_std": 685.1419, "Agent": "NPG"}, {"env_step": 1720320, "rew": 3587.7024, "rew_std": 471.134, "Agent": "NPG"}, {"env_step": 1751040, "rew": 3560.6233, "rew_std": 616.8093, "Agent": "NPG"}, {"env_step": 1781760, "rew": 3635.7303, "rew_std": 608.4666, "Agent": "NPG"}, {"env_step": 1812480, "rew": 3713.5899, "rew_std": 802.8577, "Agent": "NPG"}, {"env_step": 1843200, "rew": 3590.908, "rew_std": 557.7779, "Agent": "NPG"}, {"env_step": 1873920, "rew": 3903.7812, "rew_std": 655.7805, "Agent": "NPG"}, {"env_step": 1904640, "rew": 3770.5075, "rew_std": 645.9229, "Agent": "NPG"}, {"env_step": 1935360, "rew": 4110.3599, "rew_std": 657.3085, "Agent": "NPG"}, {"env_step": 1966080, "rew": 3851.8914, "rew_std": 745.2079, "Agent": "NPG"}, {"env_step": 1996800, "rew": 3998.0014, "rew_std": 650.7886, "Agent": "NPG"}, {"env_step": 2027520, "rew": 3739.3657, "rew_std": 715.6727, "Agent": "NPG"}, {"env_step": 2058240, "rew": 4230.5764, "rew_std": 578.5901, "Agent": "NPG"}, {"env_step": 2088960, "rew": 4140.935, "rew_std": 624.3245, "Agent": "NPG"}, {"env_step": 2119680, "rew": 4228.9808, "rew_std": 573.9637, "Agent": "NPG"}, {"env_step": 2150400, "rew": 4142.6649, "rew_std": 419.5781, "Agent": "NPG"}, {"env_step": 2181120, "rew": 4089.8147, "rew_std": 302.9776, "Agent": "NPG"}, {"env_step": 2211840, "rew": 4276.8854, "rew_std": 674.4424, "Agent": "NPG"}, {"env_step": 2242560, "rew": 4312.1654, "rew_std": 567.5481, "Agent": "NPG"}, {"env_step": 2273280, "rew": 4224.3566, "rew_std": 445.0954, "Agent": "NPG"}, {"env_step": 2304000, "rew": 3878.7632, "rew_std": 866.3551, "Agent": "NPG"}, {"env_step": 2334720, "rew": 4001.3801, "rew_std": 478.1705, "Agent": "NPG"}, {"env_step": 2365440, "rew": 4284.5594, "rew_std": 448.8121, "Agent": "NPG"}, {"env_step": 2396160, "rew": 4406.6923, "rew_std": 516.8683, "Agent": "NPG"}, {"env_step": 2426880, "rew": 4145.9032, "rew_std": 519.4268, "Agent": "NPG"}, {"env_step": 2457600, "rew": 4241.7632, "rew_std": 685.6791, "Agent": "NPG"}, {"env_step": 2488320, "rew": 4154.1219, "rew_std": 437.1567, "Agent": "NPG"}, {"env_step": 2519040, "rew": 4495.7374, "rew_std": 449.1806, "Agent": "NPG"}, {"env_step": 2549760, "rew": 4289.7487, "rew_std": 526.7084, "Agent": "NPG"}, {"env_step": 2580480, "rew": 4246.0211, "rew_std": 530.6265, "Agent": "NPG"}, {"env_step": 2611200, "rew": 4463.8985, "rew_std": 572.8797, "Agent": "NPG"}, {"env_step": 2641920, "rew": 4298.6282, "rew_std": 665.9416, "Agent": "NPG"}, {"env_step": 2672640, "rew": 4485.5394, "rew_std": 359.9158, "Agent": "NPG"}, {"env_step": 2703360, "rew": 4820.597, "rew_std": 318.5627, "Agent": "NPG"}, {"env_step": 2734080, "rew": 4540.1727, "rew_std": 412.9545, "Agent": "NPG"}, {"env_step": 2764800, "rew": 4500.1939, "rew_std": 596.1343, "Agent": "NPG"}, {"env_step": 2795520, "rew": 4664.0692, "rew_std": 426.6286, "Agent": "NPG"}, {"env_step": 2826240, "rew": 4516.467, "rew_std": 435.656, "Agent": "NPG"}, {"env_step": 2856960, "rew": 4545.139, "rew_std": 496.411, "Agent": "NPG"}, {"env_step": 2887680, "rew": 4444.0518, "rew_std": 596.6213, "Agent": "NPG"}, {"env_step": 2918400, "rew": 4658.8307, "rew_std": 652.0561, "Agent": "NPG"}, {"env_step": 2949120, "rew": 4509.4439, "rew_std": 466.3483, "Agent": "NPG"}, {"env_step": 2979840, "rew": 4589.8375, "rew_std": 414.0363, "Agent": "NPG"}, {"env_step": 3010560, "rew": 4693.0902, "rew_std": 490.4674, "Agent": "NPG"}, {"env_step": 3041280, "rew": 4678.8152, "rew_std": 313.0054, "Agent": "NPG"}, {"env_step": 3072000, "rew": 4736.6273, "rew_std": 376.7205, "Agent": "NPG"}, {"env_step": 0, "rew": 5.89, "rew_std": 1.2613, "Agent": "PPO"}, {"env_step": 30720, "rew": 991.92, "rew_std": 18.7133, "Agent": "PPO"}, {"env_step": 61440, "rew": 1000.0, "rew_std": 0.0, "Agent": "PPO"}, {"env_step": 92160, "rew": 1000.0, "rew_std": 0.0, "Agent": "PPO"}, {"env_step": 122880, "rew": 1000.0, "rew_std": 0.0, "Agent": "PPO"}, {"env_step": 153600, "rew": 1000.0, "rew_std": 0.0, "Agent": "PPO"}, {"env_step": 184320, "rew": 1000.0, "rew_std": 0.0, "Agent": "PPO"}, {"env_step": 215040, "rew": 1000.0, "rew_std": 0.0, "Agent": "PPO"}, {"env_step": 245760, "rew": 1000.0, "rew_std": 0.0, "Agent": "PPO"}, {"env_step": 276480, "rew": 1000.0, "rew_std": 0.0, "Agent": "PPO"}, {"env_step": 307200, "rew": 1000.0, "rew_std": 0.0, "Agent": "PPO"}, {"env_step": 337920, "rew": 1000.0, "rew_std": 0.0, "Agent": "PPO"}, {"env_step": 368640, "rew": 1000.0, "rew_std": 0.0, "Agent": "PPO"}, {"env_step": 399360, "rew": 1000.0, "rew_std": 0.0, "Agent": "PPO"}, {"env_step": 430080, "rew": 1000.0, "rew_std": 0.0, "Agent": "PPO"}, {"env_step": 460800, "rew": 894.23, "rew_std": 282.0792, "Agent": "PPO"}, {"env_step": 491520, "rew": 909.04, "rew_std": 184.7011, "Agent": "PPO"}, {"env_step": 522240, "rew": 953.76, "rew_std": 138.72, "Agent": "PPO"}, {"env_step": 552960, "rew": 963.62, "rew_std": 109.14, "Agent": "PPO"}, {"env_step": 583680, "rew": 979.64, "rew_std": 61.08, "Agent": "PPO"}, {"env_step": 614400, "rew": 1000.0, "rew_std": 0.0, "Agent": "PPO"}, {"env_step": 645120, "rew": 983.94, "rew_std": 48.18, "Agent": "PPO"}, {"env_step": 675840, "rew": 957.85, "rew_std": 126.45, "Agent": "PPO"}, {"env_step": 706560, "rew": 921.94, "rew_std": 234.18, "Agent": "PPO"}, {"env_step": 737280, "rew": 913.44, "rew_std": 259.68, "Agent": "PPO"}, {"env_step": 768000, "rew": 917.6, "rew_std": 247.2, "Agent": "PPO"}, {"env_step": 798720, "rew": 921.5, "rew_std": 235.5, "Agent": "PPO"}, {"env_step": 829440, "rew": 921.61, "rew_std": 235.17, "Agent": "PPO"}, {"env_step": 860160, "rew": 874.86, "rew_std": 263.0724, "Agent": "PPO"}, {"env_step": 890880, "rew": 862.17, "rew_std": 248.3223, "Agent": "PPO"}, {"env_step": 921600, "rew": 751.64, "rew_std": 379.4485, "Agent": "PPO"}, {"env_step": 952320, "rew": 817.05, "rew_std": 326.3875, "Agent": "PPO"}, {"env_step": 983040, "rew": 824.11, "rew_std": 312.4759, "Agent": "PPO"}, {"env_step": 1013760, "rew": 791.61, "rew_std": 342.7439, "Agent": "PPO"}, {"env_step": 1044480, "rew": 814.11, "rew_std": 316.6064, "Agent": "PPO"}, {"env_step": 1075200, "rew": 732.38, "rew_std": 397.0806, "Agent": "PPO"}, {"env_step": 1105920, "rew": 735.99, "rew_std": 393.5189, "Agent": "PPO"}, {"env_step": 1136640, "rew": 756.65, "rew_std": 388.4296, "Agent": "PPO"}, {"env_step": 1167360, "rew": 648.53, "rew_std": 440.111, "Agent": "PPO"}, {"env_step": 1198080, "rew": 673.59, "rew_std": 434.882, "Agent": "PPO"}, {"env_step": 1228800, "rew": 663.75, "rew_std": 435.8427, "Agent": "PPO"}, {"env_step": 1259520, "rew": 663.65, "rew_std": 435.3639, "Agent": "PPO"}, {"env_step": 1290240, "rew": 597.03, "rew_std": 439.9041, "Agent": "PPO"}, {"env_step": 1320960, "rew": 666.76, "rew_std": 430.241, "Agent": "PPO"}, {"env_step": 1351680, "rew": 695.85, "rew_std": 435.7469, "Agent": "PPO"}, {"env_step": 1382400, "rew": 614.87, "rew_std": 450.4714, "Agent": "PPO"}, {"env_step": 1413120, "rew": 514.3, "rew_std": 449.8707, "Agent": "PPO"}, {"env_step": 1443840, "rew": 504.17, "rew_std": 438.6437, "Agent": "PPO"}, {"env_step": 1474560, "rew": 486.43, "rew_std": 432.2417, "Agent": "PPO"}, {"env_step": 1505280, "rew": 492.89, "rew_std": 431.6997, "Agent": "PPO"}, {"env_step": 1536000, "rew": 505.8, "rew_std": 448.4553, "Agent": "PPO"}, {"env_step": 1566720, "rew": 504.57, "rew_std": 464.0222, "Agent": "PPO"}, {"env_step": 1597440, "rew": 478.38, "rew_std": 443.5053, "Agent": "PPO"}, {"env_step": 1628160, "rew": 499.29, "rew_std": 438.305, "Agent": "PPO"}, {"env_step": 1658880, "rew": 594.1, "rew_std": 446.8878, "Agent": "PPO"}, {"env_step": 1689600, "rew": 679.69, "rew_std": 422.8528, "Agent": "PPO"}, {"env_step": 1720320, "rew": 675.65, "rew_std": 425.7626, "Agent": "PPO"}, {"env_step": 1751040, "rew": 678.18, "rew_std": 421.0652, "Agent": "PPO"}, {"env_step": 1781760, "rew": 716.62, "rew_std": 433.3988, "Agent": "PPO"}, {"env_step": 1812480, "rew": 715.64, "rew_std": 434.7247, "Agent": "PPO"}, {"env_step": 1843200, "rew": 700.41, "rew_std": 423.6157, "Agent": "PPO"}, {"env_step": 1873920, "rew": 673.37, "rew_std": 404.6755, "Agent": "PPO"}, {"env_step": 1904640, "rew": 705.65, "rew_std": 412.9842, "Agent": "PPO"}, {"env_step": 1935360, "rew": 686.0, "rew_std": 406.3371, "Agent": "PPO"}, {"env_step": 1966080, "rew": 721.63, "rew_std": 406.8133, "Agent": "PPO"}, {"env_step": 1996800, "rew": 743.19, "rew_std": 376.0792, "Agent": "PPO"}, {"env_step": 2027520, "rew": 811.83, "rew_std": 350.2001, "Agent": "PPO"}, {"env_step": 2058240, "rew": 823.86, "rew_std": 352.3771, "Agent": "PPO"}, {"env_step": 2088960, "rew": 822.57, "rew_std": 354.871, "Agent": "PPO"}, {"env_step": 2119680, "rew": 825.47, "rew_std": 349.1095, "Agent": "PPO"}, {"env_step": 2150400, "rew": 803.09, "rew_std": 338.447, "Agent": "PPO"}, {"env_step": 2181120, "rew": 827.57, "rew_std": 344.8652, "Agent": "PPO"}, {"env_step": 2211840, "rew": 835.49, "rew_std": 329.0274, "Agent": "PPO"}, {"env_step": 2242560, "rew": 840.62, "rew_std": 319.6467, "Agent": "PPO"}, {"env_step": 2273280, "rew": 837.3, "rew_std": 310.0583, "Agent": "PPO"}, {"env_step": 2304000, "rew": 849.49, "rew_std": 305.1477, "Agent": "PPO"}, {"env_step": 2334720, "rew": 869.85, "rew_std": 250.1218, "Agent": "PPO"}, {"env_step": 2365440, "rew": 917.12, "rew_std": 248.64, "Agent": "PPO"}, {"env_step": 2396160, "rew": 915.76, "rew_std": 252.72, "Agent": "PPO"}, {"env_step": 2426880, "rew": 917.29, "rew_std": 238.4911, "Agent": "PPO"}, {"env_step": 2457600, "rew": 845.93, "rew_std": 291.2987, "Agent": "PPO"}, {"env_step": 2488320, "rew": 855.03, "rew_std": 261.2282, "Agent": "PPO"}, {"env_step": 2519040, "rew": 900.61, "rew_std": 243.8298, "Agent": "PPO"}, {"env_step": 2549760, "rew": 900.54, "rew_std": 246.8906, "Agent": "PPO"}, {"env_step": 2580480, "rew": 909.48, "rew_std": 234.424, "Agent": "PPO"}, {"env_step": 2611200, "rew": 900.49, "rew_std": 243.7414, "Agent": "PPO"}, {"env_step": 2641920, "rew": 921.6, "rew_std": 235.2, "Agent": "PPO"}, {"env_step": 2672640, "rew": 905.18, "rew_std": 237.499, "Agent": "PPO"}, {"env_step": 2703360, "rew": 925.26, "rew_std": 224.22, "Agent": "PPO"}, {"env_step": 2734080, "rew": 912.07, "rew_std": 232.4689, "Agent": "PPO"}, {"env_step": 2764800, "rew": 924.37, "rew_std": 222.5248, "Agent": "PPO"}, {"env_step": 2795520, "rew": 909.44, "rew_std": 226.5858, "Agent": "PPO"}, {"env_step": 2826240, "rew": 924.46, "rew_std": 212.8309, "Agent": "PPO"}, {"env_step": 2856960, "rew": 905.68, "rew_std": 211.3978, "Agent": "PPO"}, {"env_step": 2887680, "rew": 928.55, "rew_std": 214.35, "Agent": "PPO"}, {"env_step": 2918400, "rew": 924.14, "rew_std": 210.3213, "Agent": "PPO"}, {"env_step": 2949120, "rew": 922.3, "rew_std": 202.7814, "Agent": "PPO"}, {"env_step": 2979840, "rew": 934.52, "rew_std": 152.1107, "Agent": "PPO"}, {"env_step": 3010560, "rew": 934.61, "rew_std": 190.3107, "Agent": "PPO"}, {"env_step": 3041280, "rew": 925.86, "rew_std": 193.2455, "Agent": "PPO"}, {"env_step": 3072000, "rew": 934.13, "rew_std": 167.4007, "Agent": "PPO"}, {"env_step": 0, "rew": 6.38, "rew_std": 1.0167, "Agent": "REINFORCE"}, {"env_step": 30720, "rew": 12.69, "rew_std": 2.9851, "Agent": "REINFORCE"}, {"env_step": 61440, "rew": 32.23, "rew_std": 3.8879, "Agent": "REINFORCE"}, {"env_step": 92160, "rew": 77.27, "rew_std": 26.6147, "Agent": "REINFORCE"}, {"env_step": 122880, "rew": 148.32, "rew_std": 8.6668, "Agent": "REINFORCE"}, {"env_step": 153600, "rew": 204.35, "rew_std": 34.1581, "Agent": "REINFORCE"}, {"env_step": 184320, "rew": 241.57, "rew_std": 36.1153, "Agent": "REINFORCE"}, {"env_step": 215040, "rew": 269.84, "rew_std": 56.4856, "Agent": "REINFORCE"}, {"env_step": 245760, "rew": 292.24, "rew_std": 59.8266, "Agent": "REINFORCE"}, {"env_step": 276480, "rew": 303.53, "rew_std": 72.1803, "Agent": "REINFORCE"}, {"env_step": 307200, "rew": 366.6, "rew_std": 69.1684, "Agent": "REINFORCE"}, {"env_step": 337920, "rew": 418.87, "rew_std": 122.7315, "Agent": "REINFORCE"}, {"env_step": 368640, "rew": 540.22, "rew_std": 110.6541, "Agent": "REINFORCE"}, {"env_step": 399360, "rew": 584.33, "rew_std": 162.809, "Agent": "REINFORCE"}, {"env_step": 430080, "rew": 647.97, "rew_std": 202.2086, "Agent": "REINFORCE"}, {"env_step": 460800, "rew": 674.07, "rew_std": 184.473, "Agent": "REINFORCE"}, {"env_step": 491520, "rew": 698.35, "rew_std": 200.3386, "Agent": "REINFORCE"}, {"env_step": 522240, "rew": 708.29, "rew_std": 169.5527, "Agent": "REINFORCE"}, {"env_step": 552960, "rew": 719.61, "rew_std": 145.3608, "Agent": "REINFORCE"}, {"env_step": 583680, "rew": 736.16, "rew_std": 163.8339, "Agent": "REINFORCE"}, {"env_step": 614400, "rew": 712.89, "rew_std": 164.4856, "Agent": "REINFORCE"}, {"env_step": 645120, "rew": 758.01, "rew_std": 186.5025, "Agent": "REINFORCE"}, {"env_step": 675840, "rew": 783.12, "rew_std": 186.196, "Agent": "REINFORCE"}, {"env_step": 706560, "rew": 828.69, "rew_std": 182.6344, "Agent": "REINFORCE"}, {"env_step": 737280, "rew": 827.53, "rew_std": 195.6721, "Agent": "REINFORCE"}, {"env_step": 768000, "rew": 810.95, "rew_std": 190.6268, "Agent": "REINFORCE"}, {"env_step": 798720, "rew": 770.26, "rew_std": 172.4914, "Agent": "REINFORCE"}, {"env_step": 829440, "rew": 779.48, "rew_std": 122.8061, "Agent": "REINFORCE"}, {"env_step": 860160, "rew": 830.8, "rew_std": 146.0733, "Agent": "REINFORCE"}, {"env_step": 890880, "rew": 828.59, "rew_std": 142.9545, "Agent": "REINFORCE"}, {"env_step": 921600, "rew": 849.73, "rew_std": 96.7585, "Agent": "REINFORCE"}, {"env_step": 952320, "rew": 917.15, "rew_std": 90.8268, "Agent": "REINFORCE"}, {"env_step": 983040, "rew": 932.22, "rew_std": 76.1166, "Agent": "REINFORCE"}, {"env_step": 1013760, "rew": 879.17, "rew_std": 122.7968, "Agent": "REINFORCE"}, {"env_step": 1044480, "rew": 907.47, "rew_std": 111.0657, "Agent": "REINFORCE"}, {"env_step": 1075200, "rew": 944.47, "rew_std": 80.4061, "Agent": "REINFORCE"}, {"env_step": 1105920, "rew": 892.28, "rew_std": 127.6802, "Agent": "REINFORCE"}, {"env_step": 1136640, "rew": 861.91, "rew_std": 126.3386, "Agent": "REINFORCE"}, {"env_step": 1167360, "rew": 889.43, "rew_std": 143.7161, "Agent": "REINFORCE"}, {"env_step": 1198080, "rew": 904.98, "rew_std": 123.0215, "Agent": "REINFORCE"}, {"env_step": 1228800, "rew": 876.33, "rew_std": 171.302, "Agent": "REINFORCE"}, {"env_step": 1259520, "rew": 891.96, "rew_std": 193.7084, "Agent": "REINFORCE"}, {"env_step": 1290240, "rew": 878.39, "rew_std": 164.648, "Agent": "REINFORCE"}, {"env_step": 1320960, "rew": 861.3, "rew_std": 181.4943, "Agent": "REINFORCE"}, {"env_step": 1351680, "rew": 867.75, "rew_std": 170.6529, "Agent": "REINFORCE"}, {"env_step": 1382400, "rew": 845.37, "rew_std": 193.2622, "Agent": "REINFORCE"}, {"env_step": 1413120, "rew": 842.0, "rew_std": 193.8438, "Agent": "REINFORCE"}, {"env_step": 1443840, "rew": 820.57, "rew_std": 219.4767, "Agent": "REINFORCE"}, {"env_step": 1474560, "rew": 815.99, "rew_std": 236.7662, "Agent": "REINFORCE"}, {"env_step": 1505280, "rew": 814.27, "rew_std": 219.5517, "Agent": "REINFORCE"}, {"env_step": 1536000, "rew": 830.9, "rew_std": 232.4408, "Agent": "REINFORCE"}, {"env_step": 1566720, "rew": 827.87, "rew_std": 226.6782, "Agent": "REINFORCE"}, {"env_step": 1597440, "rew": 835.52, "rew_std": 211.3546, "Agent": "REINFORCE"}, {"env_step": 1628160, "rew": 875.83, "rew_std": 180.7828, "Agent": "REINFORCE"}, {"env_step": 1658880, "rew": 907.5, "rew_std": 145.8232, "Agent": "REINFORCE"}, {"env_step": 1689600, "rew": 918.3, "rew_std": 119.6084, "Agent": "REINFORCE"}, {"env_step": 1720320, "rew": 923.26, "rew_std": 124.8266, "Agent": "REINFORCE"}, {"env_step": 1751040, "rew": 916.52, "rew_std": 127.6408, "Agent": "REINFORCE"}, {"env_step": 1781760, "rew": 881.89, "rew_std": 168.3299, "Agent": "REINFORCE"}, {"env_step": 1812480, "rew": 854.64, "rew_std": 152.5227, "Agent": "REINFORCE"}, {"env_step": 1843200, "rew": 843.18, "rew_std": 147.2391, "Agent": "REINFORCE"}, {"env_step": 1873920, "rew": 830.27, "rew_std": 199.4989, "Agent": "REINFORCE"}, {"env_step": 1904640, "rew": 760.88, "rew_std": 235.9908, "Agent": "REINFORCE"}, {"env_step": 1935360, "rew": 752.09, "rew_std": 262.4417, "Agent": "REINFORCE"}, {"env_step": 1966080, "rew": 824.91, "rew_std": 257.6926, "Agent": "REINFORCE"}, {"env_step": 1996800, "rew": 827.51, "rew_std": 279.11, "Agent": "REINFORCE"}, {"env_step": 2027520, "rew": 830.03, "rew_std": 275.1672, "Agent": "REINFORCE"}, {"env_step": 2058240, "rew": 816.47, "rew_std": 282.6688, "Agent": "REINFORCE"}, {"env_step": 2088960, "rew": 860.69, "rew_std": 263.9615, "Agent": "REINFORCE"}, {"env_step": 2119680, "rew": 889.93, "rew_std": 165.5766, "Agent": "REINFORCE"}, {"env_step": 2150400, "rew": 914.43, "rew_std": 139.7138, "Agent": "REINFORCE"}, {"env_step": 2181120, "rew": 866.68, "rew_std": 178.7052, "Agent": "REINFORCE"}, {"env_step": 2211840, "rew": 877.44, "rew_std": 147.7871, "Agent": "REINFORCE"}, {"env_step": 2242560, "rew": 933.0, "rew_std": 124.4947, "Agent": "REINFORCE"}, {"env_step": 2273280, "rew": 907.14, "rew_std": 150.9342, "Agent": "REINFORCE"}, {"env_step": 2304000, "rew": 933.29, "rew_std": 198.7339, "Agent": "REINFORCE"}, {"env_step": 2334720, "rew": 926.73, "rew_std": 172.8165, "Agent": "REINFORCE"}, {"env_step": 2365440, "rew": 933.02, "rew_std": 180.0415, "Agent": "REINFORCE"}, {"env_step": 2396160, "rew": 927.41, "rew_std": 139.3859, "Agent": "REINFORCE"}, {"env_step": 2426880, "rew": 918.35, "rew_std": 143.5981, "Agent": "REINFORCE"}, {"env_step": 2457600, "rew": 930.83, "rew_std": 134.4429, "Agent": "REINFORCE"}, {"env_step": 2488320, "rew": 915.37, "rew_std": 192.8453, "Agent": "REINFORCE"}, {"env_step": 2519040, "rew": 888.03, "rew_std": 202.7, "Agent": "REINFORCE"}, {"env_step": 2549760, "rew": 858.34, "rew_std": 201.8235, "Agent": "REINFORCE"}, {"env_step": 2580480, "rew": 864.93, "rew_std": 204.9745, "Agent": "REINFORCE"}, {"env_step": 2611200, "rew": 858.65, "rew_std": 199.3346, "Agent": "REINFORCE"}, {"env_step": 2641920, "rew": 835.58, "rew_std": 190.5613, "Agent": "REINFORCE"}, {"env_step": 2672640, "rew": 877.95, "rew_std": 189.4195, "Agent": "REINFORCE"}, {"env_step": 2703360, "rew": 889.08, "rew_std": 136.7458, "Agent": "REINFORCE"}, {"env_step": 2734080, "rew": 875.81, "rew_std": 187.1033, "Agent": "REINFORCE"}, {"env_step": 2764800, "rew": 900.8, "rew_std": 185.6886, "Agent": "REINFORCE"}, {"env_step": 2795520, "rew": 976.84, "rew_std": 36.2903, "Agent": "REINFORCE"}, {"env_step": 2826240, "rew": 994.83, "rew_std": 14.6638, "Agent": "REINFORCE"}, {"env_step": 2856960, "rew": 982.4, "rew_std": 41.4874, "Agent": "REINFORCE"}, {"env_step": 2887680, "rew": 958.21, "rew_std": 104.2025, "Agent": "REINFORCE"}, {"env_step": 2918400, "rew": 965.74, "rew_std": 48.0941, "Agent": "REINFORCE"}, {"env_step": 2949120, "rew": 992.54, "rew_std": 15.5009, "Agent": "REINFORCE"}, {"env_step": 2979840, "rew": 999.21, "rew_std": 2.37, "Agent": "REINFORCE"}, {"env_step": 3010560, "rew": 995.32, "rew_std": 10.7451, "Agent": "REINFORCE"}, {"env_step": 3041280, "rew": 979.38, "rew_std": 42.6937, "Agent": "REINFORCE"}, {"env_step": 3072000, "rew": 964.1, "rew_std": 82.9313, "Agent": "REINFORCE"}, {"env_step": 3102720, "rew": 959.63, "rew_std": 106.8083, "Agent": "REINFORCE"}, {"env_step": 3133440, "rew": 956.85, "rew_std": 106.4687, "Agent": "REINFORCE"}, {"env_step": 3164160, "rew": 949.08, "rew_std": 126.2108, "Agent": "REINFORCE"}, {"env_step": 3194880, "rew": 959.78, "rew_std": 75.3591, "Agent": "REINFORCE"}, {"env_step": 3225600, "rew": 986.1, "rew_std": 31.7728, "Agent": "REINFORCE"}, {"env_step": 3256320, "rew": 970.11, "rew_std": 58.0504, "Agent": "REINFORCE"}, {"env_step": 3287040, "rew": 925.99, "rew_std": 169.4014, "Agent": "REINFORCE"}, {"env_step": 3317760, "rew": 937.4, "rew_std": 132.2121, "Agent": "REINFORCE"}, {"env_step": 3348480, "rew": 938.35, "rew_std": 111.87, "Agent": "REINFORCE"}, {"env_step": 3379200, "rew": 958.97, "rew_std": 76.2582, "Agent": "REINFORCE"}, {"env_step": 3409920, "rew": 973.93, "rew_std": 78.21, "Agent": "REINFORCE"}, {"env_step": 3440640, "rew": 993.26, "rew_std": 13.4807, "Agent": "REINFORCE"}, {"env_step": 3471360, "rew": 1000.0, "rew_std": 0.0, "Agent": "REINFORCE"}, {"env_step": 3502080, "rew": 991.8, "rew_std": 24.6, "Agent": "REINFORCE"}, {"env_step": 3532800, "rew": 1000.0, "rew_std": 0.0, "Agent": "REINFORCE"}, {"env_step": 3563520, "rew": 987.43, "rew_std": 37.71, "Agent": "REINFORCE"}, {"env_step": 3594240, "rew": 993.49, "rew_std": 15.1111, "Agent": "REINFORCE"}, {"env_step": 3624960, "rew": 952.32, "rew_std": 96.3679, "Agent": "REINFORCE"}, {"env_step": 3655680, "rew": 958.15, "rew_std": 86.4831, "Agent": "REINFORCE"}, {"env_step": 3686400, "rew": 996.88, "rew_std": 9.36, "Agent": "REINFORCE"}, {"env_step": 3717120, "rew": 985.05, "rew_std": 43.4356, "Agent": "REINFORCE"}, {"env_step": 3747840, "rew": 988.67, "rew_std": 19.47, "Agent": "REINFORCE"}, {"env_step": 3778560, "rew": 1000.0, "rew_std": 0.0, "Agent": "REINFORCE"}, {"env_step": 3809280, "rew": 1000.0, "rew_std": 0.0, "Agent": "REINFORCE"}, {"env_step": 3840000, "rew": 1000.0, "rew_std": 0.0, "Agent": "REINFORCE"}, {"env_step": 3870720, "rew": 1000.0, "rew_std": 0.0, "Agent": "REINFORCE"}, {"env_step": 3901440, "rew": 997.36, "rew_std": 7.92, "Agent": "REINFORCE"}, {"env_step": 3932160, "rew": 961.05, "rew_std": 89.4592, "Agent": "REINFORCE"}, {"env_step": 3962880, "rew": 996.96, "rew_std": 9.12, "Agent": "REINFORCE"}, {"env_step": 3993600, "rew": 989.29, "rew_std": 23.3794, "Agent": "REINFORCE"}, {"env_step": 4024320, "rew": 980.13, "rew_std": 44.6161, "Agent": "REINFORCE"}, {"env_step": 4055040, "rew": 936.52, "rew_std": 138.0583, "Agent": "REINFORCE"}, {"env_step": 4085760, "rew": 898.51, "rew_std": 209.1586, "Agent": "REINFORCE"}, {"env_step": 4116480, "rew": 924.11, "rew_std": 150.3964, "Agent": "REINFORCE"}, {"env_step": 4147200, "rew": 947.87, "rew_std": 120.5692, "Agent": "REINFORCE"}, {"env_step": 4177920, "rew": 985.73, "rew_std": 29.0117, "Agent": "REINFORCE"}, {"env_step": 4208640, "rew": 997.0, "rew_std": 9.0, "Agent": "REINFORCE"}, {"env_step": 4239360, "rew": 984.57, "rew_std": 46.29, "Agent": "REINFORCE"}, {"env_step": 4270080, "rew": 1000.0, "rew_std": 0.0, "Agent": "REINFORCE"}, {"env_step": 4300800, "rew": 992.52, "rew_std": 22.44, "Agent": "REINFORCE"}, {"env_step": 4331520, "rew": 996.87, "rew_std": 9.3567, "Agent": "REINFORCE"}, {"env_step": 4362240, "rew": 971.22, "rew_std": 80.9581, "Agent": "REINFORCE"}, {"env_step": 4392960, "rew": 996.96, "rew_std": 6.8386, "Agent": "REINFORCE"}, {"env_step": 4423680, "rew": 984.05, "rew_std": 47.85, "Agent": "REINFORCE"}, {"env_step": 4454400, "rew": 969.73, "rew_std": 89.3859, "Agent": "REINFORCE"}, {"env_step": 4485120, "rew": 974.38, "rew_std": 51.5344, "Agent": "REINFORCE"}, {"env_step": 4515840, "rew": 980.65, "rew_std": 38.7903, "Agent": "REINFORCE"}, {"env_step": 4546560, "rew": 994.77, "rew_std": 11.3847, "Agent": "REINFORCE"}, {"env_step": 4577280, "rew": 989.89, "rew_std": 26.3412, "Agent": "REINFORCE"}, {"env_step": 4608000, "rew": 970.11, "rew_std": 56.3644, "Agent": "REINFORCE"}, {"env_step": 4638720, "rew": 976.69, "rew_std": 59.2732, "Agent": "REINFORCE"}, {"env_step": 4669440, "rew": 986.44, "rew_std": 40.68, "Agent": "REINFORCE"}, {"env_step": 4700160, "rew": 952.71, "rew_std": 125.6674, "Agent": "REINFORCE"}, {"env_step": 4730880, "rew": 928.08, "rew_std": 111.8978, "Agent": "REINFORCE"}, {"env_step": 4761600, "rew": 896.89, "rew_std": 206.626, "Agent": "REINFORCE"}, {"env_step": 4792320, "rew": 939.65, "rew_std": 163.6885, "Agent": "REINFORCE"}, {"env_step": 4823040, "rew": 944.47, "rew_std": 116.7956, "Agent": "REINFORCE"}, {"env_step": 4853760, "rew": 985.24, "rew_std": 34.0693, "Agent": "REINFORCE"}, {"env_step": 4884480, "rew": 941.44, "rew_std": 164.5019, "Agent": "REINFORCE"}, {"env_step": 4915200, "rew": 981.87, "rew_std": 52.8745, "Agent": "REINFORCE"}, {"env_step": 4945920, "rew": 969.33, "rew_std": 66.2071, "Agent": "REINFORCE"}, {"env_step": 4976640, "rew": 969.65, "rew_std": 82.8689, "Agent": "REINFORCE"}, {"env_step": 5007360, "rew": 923.34, "rew_std": 186.0589, "Agent": "REINFORCE"}, {"env_step": 5038080, "rew": 958.08, "rew_std": 125.76, "Agent": "REINFORCE"}, {"env_step": 5068800, "rew": 964.36, "rew_std": 64.0499, "Agent": "REINFORCE"}, {"env_step": 5099520, "rew": 944.88, "rew_std": 125.0058, "Agent": "REINFORCE"}, {"env_step": 5130240, "rew": 974.54, "rew_std": 47.3084, "Agent": "REINFORCE"}, {"env_step": 5160960, "rew": 942.66, "rew_std": 116.292, "Agent": "REINFORCE"}, {"env_step": 5191680, "rew": 972.72, "rew_std": 51.941, "Agent": "REINFORCE"}, {"env_step": 5222400, "rew": 955.78, "rew_std": 89.3096, "Agent": "REINFORCE"}, {"env_step": 5253120, "rew": 931.04, "rew_std": 137.3695, "Agent": "REINFORCE"}, {"env_step": 5283840, "rew": 978.13, "rew_std": 59.9788, "Agent": "REINFORCE"}, {"env_step": 5314560, "rew": 957.28, "rew_std": 75.6188, "Agent": "REINFORCE"}, {"env_step": 5345280, "rew": 975.95, "rew_std": 68.7852, "Agent": "REINFORCE"}, {"env_step": 5376000, "rew": 966.64, "rew_std": 96.825, "Agent": "REINFORCE"}, {"env_step": 5406720, "rew": 979.06, "rew_std": 62.82, "Agent": "REINFORCE"}, {"env_step": 5437440, "rew": 968.61, "rew_std": 86.5537, "Agent": "REINFORCE"}, {"env_step": 5468160, "rew": 976.44, "rew_std": 70.68, "Agent": "REINFORCE"}, {"env_step": 5498880, "rew": 958.38, "rew_std": 121.4329, "Agent": "REINFORCE"}, {"env_step": 5529600, "rew": 955.82, "rew_std": 88.6493, "Agent": "REINFORCE"}, {"env_step": 5560320, "rew": 946.08, "rew_std": 112.7584, "Agent": "REINFORCE"}, {"env_step": 5591040, "rew": 961.89, "rew_std": 60.2556, "Agent": "REINFORCE"}, {"env_step": 5621760, "rew": 940.04, "rew_std": 120.9129, "Agent": "REINFORCE"}, {"env_step": 5652480, "rew": 899.66, "rew_std": 236.9832, "Agent": "REINFORCE"}, {"env_step": 5683200, "rew": 931.92, "rew_std": 116.4014, "Agent": "REINFORCE"}, {"env_step": 5713920, "rew": 945.26, "rew_std": 146.3737, "Agent": "REINFORCE"}, {"env_step": 5744640, "rew": 948.53, "rew_std": 139.2984, "Agent": "REINFORCE"}, {"env_step": 5775360, "rew": 935.82, "rew_std": 163.0163, "Agent": "REINFORCE"}, {"env_step": 5806080, "rew": 941.58, "rew_std": 133.742, "Agent": "REINFORCE"}, {"env_step": 5836800, "rew": 946.76, "rew_std": 106.4974, "Agent": "REINFORCE"}, {"env_step": 5867520, "rew": 962.57, "rew_std": 79.0669, "Agent": "REINFORCE"}, {"env_step": 5898240, "rew": 981.65, "rew_std": 55.05, "Agent": "REINFORCE"}, {"env_step": 5928960, "rew": 972.05, "rew_std": 66.1885, "Agent": "REINFORCE"}, {"env_step": 5959680, "rew": 984.14, "rew_std": 47.58, "Agent": "REINFORCE"}, {"env_step": 5990400, "rew": 967.55, "rew_std": 88.655, "Agent": "REINFORCE"}, {"env_step": 6021120, "rew": 971.08, "rew_std": 86.76, "Agent": "REINFORCE"}, {"env_step": 6051840, "rew": 968.22, "rew_std": 95.34, "Agent": "REINFORCE"}, {"env_step": 6082560, "rew": 977.34, "rew_std": 67.98, "Agent": "REINFORCE"}, {"env_step": 6113280, "rew": 960.5, "rew_std": 118.5, "Agent": "REINFORCE"}, {"env_step": 6144000, "rew": 973.86, "rew_std": 78.42, "Agent": "REINFORCE"}, {"env_step": 6174720, "rew": 969.46, "rew_std": 58.3955, "Agent": "REINFORCE"}, {"env_step": 6205440, "rew": 979.65, "rew_std": 41.8987, "Agent": "REINFORCE"}, {"env_step": 6236160, "rew": 965.02, "rew_std": 99.5217, "Agent": "REINFORCE"}, {"env_step": 6266880, "rew": 971.33, "rew_std": 47.5716, "Agent": "REINFORCE"}, {"env_step": 6297600, "rew": 974.53, "rew_std": 38.7326, "Agent": "REINFORCE"}, {"env_step": 6328320, "rew": 977.68, "rew_std": 49.2839, "Agent": "REINFORCE"}, {"env_step": 6359040, "rew": 943.14, "rew_std": 137.6136, "Agent": "REINFORCE"}, {"env_step": 6389760, "rew": 929.24, "rew_std": 140.7475, "Agent": "REINFORCE"}, {"env_step": 6420480, "rew": 945.01, "rew_std": 125.1647, "Agent": "REINFORCE"}, {"env_step": 6451200, "rew": 985.98, "rew_std": 29.0759, "Agent": "REINFORCE"}, {"env_step": 6481920, "rew": 973.27, "rew_std": 54.1206, "Agent": "REINFORCE"}, {"env_step": 6512640, "rew": 993.2, "rew_std": 20.4, "Agent": "REINFORCE"}, {"env_step": 6543360, "rew": 995.22, "rew_std": 14.34, "Agent": "REINFORCE"}, {"env_step": 6574080, "rew": 983.8, "rew_std": 48.6, "Agent": "REINFORCE"}, {"env_step": 6604800, "rew": 977.69, "rew_std": 66.93, "Agent": "REINFORCE"}, {"env_step": 6635520, "rew": 970.33, "rew_std": 89.01, "Agent": "REINFORCE"}, {"env_step": 6666240, "rew": 928.35, "rew_std": 143.5004, "Agent": "REINFORCE"}, {"env_step": 6696960, "rew": 887.07, "rew_std": 264.9795, "Agent": "REINFORCE"}, {"env_step": 6727680, "rew": 912.96, "rew_std": 165.7926, "Agent": "REINFORCE"}, {"env_step": 6758400, "rew": 944.63, "rew_std": 112.7822, "Agent": "REINFORCE"}, {"env_step": 6789120, "rew": 972.56, "rew_std": 59.3623, "Agent": "REINFORCE"}, {"env_step": 6819840, "rew": 951.99, "rew_std": 100.6198, "Agent": "REINFORCE"}, {"env_step": 6850560, "rew": 970.37, "rew_std": 88.89, "Agent": "REINFORCE"}, {"env_step": 6881280, "rew": 959.42, "rew_std": 102.4488, "Agent": "REINFORCE"}, {"env_step": 6912000, "rew": 976.17, "rew_std": 71.49, "Agent": "REINFORCE"}, {"env_step": 6942720, "rew": 986.69, "rew_std": 29.621, "Agent": "REINFORCE"}, {"env_step": 6973440, "rew": 967.66, "rew_std": 75.8986, "Agent": "REINFORCE"}, {"env_step": 7004160, "rew": 974.04, "rew_std": 71.0949, "Agent": "REINFORCE"}, {"env_step": 7034880, "rew": 971.19, "rew_std": 63.3525, "Agent": "REINFORCE"}, {"env_step": 7065600, "rew": 964.4, "rew_std": 70.0223, "Agent": "REINFORCE"}, {"env_step": 7096320, "rew": 975.77, "rew_std": 42.3078, "Agent": "REINFORCE"}, {"env_step": 7127040, "rew": 953.71, "rew_std": 107.6088, "Agent": "REINFORCE"}, {"env_step": 7157760, "rew": 946.54, "rew_std": 160.38, "Agent": "REINFORCE"}, {"env_step": 7188480, "rew": 945.85, "rew_std": 156.6389, "Agent": "REINFORCE"}, {"env_step": 7219200, "rew": 986.76, "rew_std": 28.1261, "Agent": "REINFORCE"}, {"env_step": 7249920, "rew": 969.07, "rew_std": 79.0497, "Agent": "REINFORCE"}, {"env_step": 7280640, "rew": 964.18, "rew_std": 71.6487, "Agent": "REINFORCE"}, {"env_step": 7311360, "rew": 973.94, "rew_std": 66.5413, "Agent": "REINFORCE"}, {"env_step": 7342080, "rew": 967.18, "rew_std": 64.9514, "Agent": "REINFORCE"}, {"env_step": 7372800, "rew": 987.3, "rew_std": 38.1, "Agent": "REINFORCE"}, {"env_step": 7403520, "rew": 987.18, "rew_std": 26.4402, "Agent": "REINFORCE"}, {"env_step": 7434240, "rew": 988.15, "rew_std": 27.092, "Agent": "REINFORCE"}, {"env_step": 7464960, "rew": 997.34, "rew_std": 7.98, "Agent": "REINFORCE"}, {"env_step": 7495680, "rew": 989.45, "rew_std": 22.1128, "Agent": "REINFORCE"}, {"env_step": 7526400, "rew": 976.87, "rew_std": 32.5971, "Agent": "REINFORCE"}, {"env_step": 7557120, "rew": 990.95, "rew_std": 19.0453, "Agent": "REINFORCE"}, {"env_step": 7587840, "rew": 985.18, "rew_std": 44.46, "Agent": "REINFORCE"}, {"env_step": 7618560, "rew": 986.04, "rew_std": 27.1982, "Agent": "REINFORCE"}, {"env_step": 7649280, "rew": 996.89, "rew_std": 9.33, "Agent": "REINFORCE"}, {"env_step": 7680000, "rew": 995.35, "rew_std": 13.95, "Agent": "REINFORCE"}, {"env_step": 7710720, "rew": 993.68, "rew_std": 13.2519, "Agent": "REINFORCE"}, {"env_step": 7741440, "rew": 995.56, "rew_std": 13.32, "Agent": "REINFORCE"}, {"env_step": 7772160, "rew": 994.58, "rew_std": 16.26, "Agent": "REINFORCE"}, {"env_step": 7802880, "rew": 988.5, "rew_std": 19.8029, "Agent": "REINFORCE"}, {"env_step": 7833600, "rew": 983.51, "rew_std": 29.6331, "Agent": "REINFORCE"}, {"env_step": 7864320, "rew": 986.0, "rew_std": 35.8798, "Agent": "REINFORCE"}, {"env_step": 7895040, "rew": 999.01, "rew_std": 2.5046, "Agent": "REINFORCE"}, {"env_step": 7925760, "rew": 991.12, "rew_std": 19.6687, "Agent": "REINFORCE"}, {"env_step": 7956480, "rew": 995.02, "rew_std": 7.7363, "Agent": "REINFORCE"}, {"env_step": 7987200, "rew": 987.84, "rew_std": 16.5385, "Agent": "REINFORCE"}, {"env_step": 8017920, "rew": 998.27, "rew_std": 3.4658, "Agent": "REINFORCE"}, {"env_step": 8048640, "rew": 994.55, "rew_std": 11.6745, "Agent": "REINFORCE"}, {"env_step": 8079360, "rew": 997.05, "rew_std": 8.85, "Agent": "REINFORCE"}, {"env_step": 8110080, "rew": 996.69, "rew_std": 9.93, "Agent": "REINFORCE"}, {"env_step": 8140800, "rew": 988.87, "rew_std": 26.236, "Agent": "REINFORCE"}, {"env_step": 8171520, "rew": 998.81, "rew_std": 3.57, "Agent": "REINFORCE"}, {"env_step": 8202240, "rew": 987.15, "rew_std": 33.6569, "Agent": "REINFORCE"}, {"env_step": 8232960, "rew": 982.36, "rew_std": 27.6911, "Agent": "REINFORCE"}, {"env_step": 8263680, "rew": 989.91, "rew_std": 18.1475, "Agent": "REINFORCE"}, {"env_step": 8294400, "rew": 992.33, "rew_std": 15.7267, "Agent": "REINFORCE"}, {"env_step": 8325120, "rew": 984.41, "rew_std": 26.5234, "Agent": "REINFORCE"}, {"env_step": 8355840, "rew": 989.7, "rew_std": 26.1073, "Agent": "REINFORCE"}, {"env_step": 8386560, "rew": 994.3, "rew_std": 15.6883, "Agent": "REINFORCE"}, {"env_step": 8417280, "rew": 993.42, "rew_std": 15.0441, "Agent": "REINFORCE"}, {"env_step": 8448000, "rew": 995.75, "rew_std": 12.75, "Agent": "REINFORCE"}, {"env_step": 8478720, "rew": 987.79, "rew_std": 36.63, "Agent": "REINFORCE"}, {"env_step": 8509440, "rew": 990.6, "rew_std": 28.2, "Agent": "REINFORCE"}, {"env_step": 8540160, "rew": 987.92, "rew_std": 24.16, "Agent": "REINFORCE"}, {"env_step": 8570880, "rew": 993.01, "rew_std": 20.97, "Agent": "REINFORCE"}, {"env_step": 8601600, "rew": 991.45, "rew_std": 25.65, "Agent": "REINFORCE"}, {"env_step": 8632320, "rew": 988.16, "rew_std": 24.1036, "Agent": "REINFORCE"}, {"env_step": 8663040, "rew": 989.99, "rew_std": 27.2537, "Agent": "REINFORCE"}, {"env_step": 8693760, "rew": 980.57, "rew_std": 52.776, "Agent": "REINFORCE"}, {"env_step": 8724480, "rew": 979.38, "rew_std": 58.5658, "Agent": "REINFORCE"}, {"env_step": 8755200, "rew": 981.04, "rew_std": 45.9426, "Agent": "REINFORCE"}, {"env_step": 8785920, "rew": 988.11, "rew_std": 29.5302, "Agent": "REINFORCE"}, {"env_step": 8816640, "rew": 987.7, "rew_std": 24.7459, "Agent": "REINFORCE"}, {"env_step": 8847360, "rew": 979.86, "rew_std": 60.42, "Agent": "REINFORCE"}, {"env_step": 8878080, "rew": 977.88, "rew_std": 64.8739, "Agent": "REINFORCE"}, {"env_step": 8908800, "rew": 959.86, "rew_std": 108.7084, "Agent": "REINFORCE"}, {"env_step": 8939520, "rew": 970.0, "rew_std": 68.9139, "Agent": "REINFORCE"}, {"env_step": 8970240, "rew": 964.68, "rew_std": 74.6974, "Agent": "REINFORCE"}, {"env_step": 9000960, "rew": 990.37, "rew_std": 28.89, "Agent": "REINFORCE"}, {"env_step": 9031680, "rew": 979.94, "rew_std": 50.1546, "Agent": "REINFORCE"}, {"env_step": 9062400, "rew": 982.1, "rew_std": 52.7733, "Agent": "REINFORCE"}, {"env_step": 9093120, "rew": 986.99, "rew_std": 39.03, "Agent": "REINFORCE"}, {"env_step": 9123840, "rew": 989.79, "rew_std": 21.3974, "Agent": "REINFORCE"}, {"env_step": 9154560, "rew": 981.83, "rew_std": 52.1542, "Agent": "REINFORCE"}, {"env_step": 9185280, "rew": 982.49, "rew_std": 49.1926, "Agent": "REINFORCE"}, {"env_step": 9216000, "rew": 988.71, "rew_std": 33.87, "Agent": "REINFORCE"}, {"env_step": 9246720, "rew": 979.35, "rew_std": 34.6324, "Agent": "REINFORCE"}, {"env_step": 9277440, "rew": 974.84, "rew_std": 68.521, "Agent": "REINFORCE"}, {"env_step": 9308160, "rew": 985.45, "rew_std": 29.2879, "Agent": "REINFORCE"}, {"env_step": 9338880, "rew": 981.71, "rew_std": 40.8931, "Agent": "REINFORCE"}, {"env_step": 9369600, "rew": 982.45, "rew_std": 44.3861, "Agent": "REINFORCE"}, {"env_step": 9400320, "rew": 975.6, "rew_std": 50.5635, "Agent": "REINFORCE"}, {"env_step": 9431040, "rew": 954.35, "rew_std": 51.0388, "Agent": "REINFORCE"}, {"env_step": 9461760, "rew": 982.39, "rew_std": 40.8217, "Agent": "REINFORCE"}, {"env_step": 9492480, "rew": 977.28, "rew_std": 35.9271, "Agent": "REINFORCE"}, {"env_step": 9523200, "rew": 984.04, "rew_std": 32.9361, "Agent": "REINFORCE"}, {"env_step": 9553920, "rew": 997.36, "rew_std": 6.4032, "Agent": "REINFORCE"}, {"env_step": 9584640, "rew": 996.45, "rew_std": 10.65, "Agent": "REINFORCE"}, {"env_step": 9615360, "rew": 998.23, "rew_std": 3.6458, "Agent": "REINFORCE"}, {"env_step": 9646080, "rew": 998.04, "rew_std": 5.88, "Agent": "REINFORCE"}, {"env_step": 9676800, "rew": 1000.0, "rew_std": 0.0, "Agent": "REINFORCE"}, {"env_step": 9707520, "rew": 997.27, "rew_std": 8.19, "Agent": "REINFORCE"}, {"env_step": 9738240, "rew": 997.57, "rew_std": 4.8625, "Agent": "REINFORCE"}, {"env_step": 9768960, "rew": 998.22, "rew_std": 4.8606, "Agent": "REINFORCE"}, {"env_step": 9799680, "rew": 998.18, "rew_std": 5.46, "Agent": "REINFORCE"}, {"env_step": 9830400, "rew": 991.26, "rew_std": 25.9214, "Agent": "REINFORCE"}, {"env_step": 9861120, "rew": 995.78, "rew_std": 12.66, "Agent": "REINFORCE"}, {"env_step": 9891840, "rew": 993.67, "rew_std": 16.3826, "Agent": "REINFORCE"}, {"env_step": 9922560, "rew": 985.09, "rew_std": 30.4634, "Agent": "REINFORCE"}, {"env_step": 9953280, "rew": 993.93, "rew_std": 11.1788, "Agent": "REINFORCE"}, {"env_step": 9984000, "rew": 987.52, "rew_std": 28.9694, "Agent": "REINFORCE"}, {"env_step": 10014720, "rew": 993.16, "rew_std": 15.6074, "Agent": "REINFORCE"}, {"env_step": 10045440, "rew": 991.35, "rew_std": 17.309, "Agent": "REINFORCE"}, {"env_step": 10076160, "rew": 999.52, "rew_std": 1.2246, "Agent": "REINFORCE"}, {"env_step": 10106880, "rew": 1000.0, "rew_std": 0.0, "Agent": "REINFORCE"}, {"env_step": 10137600, "rew": 990.79, "rew_std": 21.6526, "Agent": "REINFORCE"}, {"env_step": 10168320, "rew": 998.24, "rew_std": 5.28, "Agent": "REINFORCE"}, {"env_step": 10199040, "rew": 982.93, "rew_std": 32.6478, "Agent": "REINFORCE"}, {"env_step": 10229760, "rew": 991.54, "rew_std": 17.0852, "Agent": "REINFORCE"}, {"env_step": 0, "rew": 5.85, "rew_std": 1.136, "Agent": "A2C"}, {"env_step": 30000, "rew": 88.42, "rew_std": 63.9144, "Agent": "A2C"}, {"env_step": 60000, "rew": 806.95, "rew_std": 282.5109, "Agent": "A2C"}, {"env_step": 90000, "rew": 938.33, "rew_std": 110.1344, "Agent": "A2C"}, {"env_step": 120000, "rew": 997.57, "rew_std": 7.29, "Agent": "A2C"}, {"env_step": 150000, "rew": 958.72, "rew_std": 123.84, "Agent": "A2C"}, {"env_step": 180000, "rew": 1000.0, "rew_std": 0.0, "Agent": "A2C"}, {"env_step": 210000, "rew": 1000.0, "rew_std": 0.0, "Agent": "A2C"}, {"env_step": 240000, "rew": 1000.0, "rew_std": 0.0, "Agent": "A2C"}, {"env_step": 270000, "rew": 1000.0, "rew_std": 0.0, "Agent": "A2C"}, {"env_step": 300000, "rew": 1000.0, "rew_std": 0.0, "Agent": "A2C"}, {"env_step": 330000, "rew": 1000.0, "rew_std": 0.0, "Agent": "A2C"}, {"env_step": 360000, "rew": 985.62, "rew_std": 43.14, "Agent": "A2C"}, {"env_step": 390000, "rew": 1000.0, "rew_std": 0.0, "Agent": "A2C"}, {"env_step": 420000, "rew": 1000.0, "rew_std": 0.0, "Agent": "A2C"}, {"env_step": 450000, "rew": 1000.0, "rew_std": 0.0, "Agent": "A2C"}, {"env_step": 480000, "rew": 908.1, "rew_std": 275.7, "Agent": "A2C"}, {"env_step": 510000, "rew": 950.15, "rew_std": 149.55, "Agent": "A2C"}, {"env_step": 540000, "rew": 1000.0, "rew_std": 0.0, "Agent": "A2C"}, {"env_step": 570000, "rew": 1000.0, "rew_std": 0.0, "Agent": "A2C"}, {"env_step": 600000, "rew": 1000.0, "rew_std": 0.0, "Agent": "A2C"}, {"env_step": 630000, "rew": 1000.0, "rew_std": 0.0, "Agent": "A2C"}, {"env_step": 660000, "rew": 987.44, "rew_std": 37.68, "Agent": "A2C"}, {"env_step": 690000, "rew": 985.83, "rew_std": 30.9024, "Agent": "A2C"}, {"env_step": 720000, "rew": 1000.0, "rew_std": 0.0, "Agent": "A2C"}, {"env_step": 750000, "rew": 1000.0, "rew_std": 0.0, "Agent": "A2C"}, {"env_step": 780000, "rew": 916.63, "rew_std": 240.8891, "Agent": "A2C"}, {"env_step": 810000, "rew": 1000.0, "rew_std": 0.0, "Agent": "A2C"}, {"env_step": 840000, "rew": 1000.0, "rew_std": 0.0, "Agent": "A2C"}, {"env_step": 870000, "rew": 980.07, "rew_std": 59.79, "Agent": "A2C"}, {"env_step": 900000, "rew": 990.03, "rew_std": 29.91, "Agent": "A2C"}, {"env_step": 930000, "rew": 982.98, "rew_std": 51.06, "Agent": "A2C"}, {"env_step": 960000, "rew": 1000.0, "rew_std": 0.0, "Agent": "A2C"}, {"env_step": 990000, "rew": 997.05, "rew_std": 8.85, "Agent": "A2C"}, {"env_step": 1020000, "rew": 1000.0, "rew_std": 0.0, "Agent": "A2C"}, {"env_step": 1050000, "rew": 990.3, "rew_std": 29.1, "Agent": "A2C"}, {"env_step": 1080000, "rew": 997.45, "rew_std": 7.65, "Agent": "A2C"}, {"env_step": 1110000, "rew": 1000.0, "rew_std": 0.0, "Agent": "A2C"}, {"env_step": 1140000, "rew": 1000.0, "rew_std": 0.0, "Agent": "A2C"}, {"env_step": 1170000, "rew": 998.3, "rew_std": 5.1, "Agent": "A2C"}, {"env_step": 1200000, "rew": 996.27, "rew_std": 11.19, "Agent": "A2C"}, {"env_step": 1230000, "rew": 991.3, "rew_std": 26.1, "Agent": "A2C"}, {"env_step": 1260000, "rew": 1000.0, "rew_std": 0.0, "Agent": "A2C"}, {"env_step": 1290000, "rew": 988.05, "rew_std": 35.85, "Agent": "A2C"}, {"env_step": 1320000, "rew": 990.15, "rew_std": 29.55, "Agent": "A2C"}, {"env_step": 1350000, "rew": 966.65, "rew_std": 62.7151, "Agent": "A2C"}, {"env_step": 1380000, "rew": 990.04, "rew_std": 29.88, "Agent": "A2C"}, {"env_step": 1410000, "rew": 965.01, "rew_std": 41.3867, "Agent": "A2C"}, {"env_step": 1440000, "rew": 988.5, "rew_std": 24.6456, "Agent": "A2C"}, {"env_step": 1470000, "rew": 1000.0, "rew_std": 0.0, "Agent": "A2C"}, {"env_step": 1500000, "rew": 961.3, "rew_std": 77.545, "Agent": "A2C"}, {"env_step": 1530000, "rew": 982.48, "rew_std": 52.56, "Agent": "A2C"}, {"env_step": 1560000, "rew": 983.44, "rew_std": 32.7019, "Agent": "A2C"}, {"env_step": 1590000, "rew": 990.04, "rew_std": 29.88, "Agent": "A2C"}, {"env_step": 1620000, "rew": 1000.0, "rew_std": 0.0, "Agent": "A2C"}, {"env_step": 1650000, "rew": 990.52, "rew_std": 28.44, "Agent": "A2C"}, {"env_step": 1680000, "rew": 996.82, "rew_std": 9.54, "Agent": "A2C"}, {"env_step": 1710000, "rew": 990.35, "rew_std": 28.95, "Agent": "A2C"}, {"env_step": 1740000, "rew": 993.4, "rew_std": 19.8, "Agent": "A2C"}, {"env_step": 1770000, "rew": 986.9, "rew_std": 20.1303, "Agent": "A2C"}, {"env_step": 1800000, "rew": 980.74, "rew_std": 31.9232, "Agent": "A2C"}, {"env_step": 1830000, "rew": 999.68, "rew_std": 0.96, "Agent": "A2C"}, {"env_step": 1860000, "rew": 1000.0, "rew_std": 0.0, "Agent": "A2C"}, {"env_step": 1890000, "rew": 998.68, "rew_std": 3.96, "Agent": "A2C"}, {"env_step": 1920000, "rew": 990.71, "rew_std": 17.068, "Agent": "A2C"}, {"env_step": 1950000, "rew": 976.31, "rew_std": 32.7669, "Agent": "A2C"}, {"env_step": 1980000, "rew": 985.47, "rew_std": 26.0654, "Agent": "A2C"}, {"env_step": 2010000, "rew": 990.07, "rew_std": 19.927, "Agent": "A2C"}, {"env_step": 2040000, "rew": 973.07, "rew_std": 43.6664, "Agent": "A2C"}, {"env_step": 2070000, "rew": 960.61, "rew_std": 48.4627, "Agent": "A2C"}, {"env_step": 2100000, "rew": 968.58, "rew_std": 34.9946, "Agent": "A2C"}, {"env_step": 2130000, "rew": 986.28, "rew_std": 29.8019, "Agent": "A2C"}, {"env_step": 2160000, "rew": 961.58, "rew_std": 74.5038, "Agent": "A2C"}, {"env_step": 2190000, "rew": 994.86, "rew_std": 15.42, "Agent": "A2C"}, {"env_step": 2220000, "rew": 990.98, "rew_std": 18.0409, "Agent": "A2C"}, {"env_step": 2250000, "rew": 979.94, "rew_std": 31.9933, "Agent": "A2C"}, {"env_step": 2280000, "rew": 937.58, "rew_std": 86.2823, "Agent": "A2C"}, {"env_step": 2310000, "rew": 990.43, "rew_std": 21.5514, "Agent": "A2C"}, {"env_step": 2340000, "rew": 986.97, "rew_std": 26.871, "Agent": "A2C"}, {"env_step": 2370000, "rew": 986.95, "rew_std": 28.0129, "Agent": "A2C"}, {"env_step": 2400000, "rew": 965.02, "rew_std": 59.0354, "Agent": "A2C"}, {"env_step": 2430000, "rew": 958.08, "rew_std": 44.2822, "Agent": "A2C"}, {"env_step": 2460000, "rew": 988.03, "rew_std": 22.285, "Agent": "A2C"}, {"env_step": 2490000, "rew": 965.07, "rew_std": 40.4661, "Agent": "A2C"}, {"env_step": 2520000, "rew": 971.9, "rew_std": 42.5035, "Agent": "A2C"}, {"env_step": 2550000, "rew": 973.6, "rew_std": 36.2035, "Agent": "A2C"}, {"env_step": 2580000, "rew": 982.17, "rew_std": 27.2857, "Agent": "A2C"}, {"env_step": 2610000, "rew": 997.66, "rew_std": 5.784, "Agent": "A2C"}, {"env_step": 2640000, "rew": 997.64, "rew_std": 7.08, "Agent": "A2C"}, {"env_step": 2670000, "rew": 952.95, "rew_std": 64.1054, "Agent": "A2C"}, {"env_step": 2700000, "rew": 1000.0, "rew_std": 0.0, "Agent": "A2C"}, {"env_step": 2730000, "rew": 993.46, "rew_std": 19.62, "Agent": "A2C"}, {"env_step": 2760000, "rew": 972.1, "rew_std": 42.7232, "Agent": "A2C"}, {"env_step": 2790000, "rew": 971.05, "rew_std": 49.9594, "Agent": "A2C"}, {"env_step": 2820000, "rew": 980.16, "rew_std": 34.1031, "Agent": "A2C"}, {"env_step": 2850000, "rew": 1000.0, "rew_std": 0.0, "Agent": "A2C"}, {"env_step": 2880000, "rew": 987.71, "rew_std": 15.4241, "Agent": "A2C"}, {"env_step": 2910000, "rew": 978.08, "rew_std": 52.676, "Agent": "A2C"}, {"env_step": 2940000, "rew": 982.23, "rew_std": 29.1027, "Agent": "A2C"}, {"env_step": 2970000, "rew": 965.08, "rew_std": 58.7017, "Agent": "A2C"}, {"env_step": 3000000, "rew": 980.32, "rew_std": 28.4113, "Agent": "A2C"}] \ No newline at end of file diff --git a/examples/mujoco/benchmark/Reacher-v2/result.json b/examples/mujoco/benchmark/Reacher-v2/result.json new file mode 100644 index 0000000..2ac9789 --- /dev/null +++ b/examples/mujoco/benchmark/Reacher-v2/result.json @@ -0,0 +1 @@ +[{"env_step": 0, "rew": -27.3232, "rew_std": 23.4601, "Agent": "SAC"}, {"env_step": 5000, "rew": -6.8932, "rew_std": 0.4332, "Agent": "SAC"}, {"env_step": 10000, "rew": -5.4257, "rew_std": 0.375, "Agent": "SAC"}, {"env_step": 15000, "rew": -4.6999, "rew_std": 0.3618, "Agent": "SAC"}, {"env_step": 20000, "rew": -3.9823, "rew_std": 0.2413, "Agent": "SAC"}, {"env_step": 25000, "rew": -4.8923, "rew_std": 0.2754, "Agent": "SAC"}, {"env_step": 30000, "rew": -3.9187, "rew_std": 0.2005, "Agent": "SAC"}, {"env_step": 35000, "rew": -4.0653, "rew_std": 0.1962, "Agent": "SAC"}, {"env_step": 40000, "rew": -4.2562, "rew_std": 0.2526, "Agent": "SAC"}, {"env_step": 45000, "rew": -4.049, "rew_std": 0.4252, "Agent": "SAC"}, {"env_step": 50000, "rew": -4.1731, "rew_std": 0.4454, "Agent": "SAC"}, {"env_step": 55000, "rew": -4.2256, "rew_std": 0.2316, "Agent": "SAC"}, {"env_step": 60000, "rew": -3.7714, "rew_std": 0.2894, "Agent": "SAC"}, {"env_step": 65000, "rew": -3.5403, "rew_std": 0.1666, "Agent": "SAC"}, {"env_step": 70000, "rew": -4.2174, "rew_std": 0.2763, "Agent": "SAC"}, {"env_step": 75000, "rew": -3.7467, "rew_std": 0.1565, "Agent": "SAC"}, {"env_step": 80000, "rew": -3.914, "rew_std": 0.2841, "Agent": "SAC"}, {"env_step": 85000, "rew": -3.8556, "rew_std": 0.1279, "Agent": "SAC"}, {"env_step": 90000, "rew": -4.6196, "rew_std": 0.3522, "Agent": "SAC"}, {"env_step": 95000, "rew": -4.0472, "rew_std": 0.1998, "Agent": "SAC"}, {"env_step": 100000, "rew": -4.2188, "rew_std": 0.2045, "Agent": "SAC"}, {"env_step": 105000, "rew": -4.2111, "rew_std": 0.3011, "Agent": "SAC"}, {"env_step": 110000, "rew": -4.4007, "rew_std": 0.307, "Agent": "SAC"}, {"env_step": 115000, "rew": -3.5302, "rew_std": 0.1679, "Agent": "SAC"}, {"env_step": 120000, "rew": -3.2939, "rew_std": 0.1955, "Agent": "SAC"}, {"env_step": 125000, "rew": -3.5366, "rew_std": 0.2136, "Agent": "SAC"}, {"env_step": 130000, "rew": -4.0446, "rew_std": 0.1986, "Agent": "SAC"}, {"env_step": 135000, "rew": -3.8398, "rew_std": 0.4702, "Agent": "SAC"}, {"env_step": 140000, "rew": -4.0787, "rew_std": 0.3272, "Agent": "SAC"}, {"env_step": 145000, "rew": -4.5688, "rew_std": 0.1852, "Agent": "SAC"}, {"env_step": 150000, "rew": -4.0183, "rew_std": 0.1741, "Agent": "SAC"}, {"env_step": 155000, "rew": -3.9579, "rew_std": 0.3947, "Agent": "SAC"}, {"env_step": 160000, "rew": -3.9063, "rew_std": 0.2715, "Agent": "SAC"}, {"env_step": 165000, "rew": -4.0287, "rew_std": 0.3581, "Agent": "SAC"}, {"env_step": 170000, "rew": -4.0284, "rew_std": 0.1758, "Agent": "SAC"}, {"env_step": 175000, "rew": -3.9873, "rew_std": 0.2332, "Agent": "SAC"}, {"env_step": 180000, "rew": -4.0801, "rew_std": 0.1964, "Agent": "SAC"}, {"env_step": 185000, "rew": -3.5858, "rew_std": 0.2554, "Agent": "SAC"}, {"env_step": 190000, "rew": -3.8319, "rew_std": 0.2312, "Agent": "SAC"}, {"env_step": 195000, "rew": -3.54, "rew_std": 0.2315, "Agent": "SAC"}, {"env_step": 200000, "rew": -3.5242, "rew_std": 0.2424, "Agent": "SAC"}, {"env_step": 205000, "rew": -3.9358, "rew_std": 0.2275, "Agent": "SAC"}, {"env_step": 210000, "rew": -4.133, "rew_std": 0.1878, "Agent": "SAC"}, {"env_step": 215000, "rew": -4.2018, "rew_std": 0.1802, "Agent": "SAC"}, {"env_step": 220000, "rew": -3.6076, "rew_std": 0.2573, "Agent": "SAC"}, {"env_step": 225000, "rew": -4.0694, "rew_std": 0.1866, "Agent": "SAC"}, {"env_step": 230000, "rew": -4.0068, "rew_std": 0.171, "Agent": "SAC"}, {"env_step": 235000, "rew": -3.1935, "rew_std": 0.2678, "Agent": "SAC"}, {"env_step": 240000, "rew": -4.2044, "rew_std": 0.2314, "Agent": "SAC"}, {"env_step": 245000, "rew": -3.9757, "rew_std": 0.2236, "Agent": "SAC"}, {"env_step": 250000, "rew": -3.853, "rew_std": 0.2918, "Agent": "SAC"}, {"env_step": 255000, "rew": -3.3434, "rew_std": 0.2674, "Agent": "SAC"}, {"env_step": 260000, "rew": -3.803, "rew_std": 0.076, "Agent": "SAC"}, {"env_step": 265000, "rew": -3.6264, "rew_std": 0.2275, "Agent": "SAC"}, {"env_step": 270000, "rew": -3.7682, "rew_std": 0.2303, "Agent": "SAC"}, {"env_step": 275000, "rew": -3.6978, "rew_std": 0.2235, "Agent": "SAC"}, {"env_step": 280000, "rew": -3.7764, "rew_std": 0.2653, "Agent": "SAC"}, {"env_step": 285000, "rew": -4.1724, "rew_std": 0.1296, "Agent": "SAC"}, {"env_step": 290000, "rew": -3.4387, "rew_std": 0.1367, "Agent": "SAC"}, {"env_step": 295000, "rew": -3.498, "rew_std": 0.1265, "Agent": "SAC"}, {"env_step": 300000, "rew": -3.6395, "rew_std": 0.4138, "Agent": "SAC"}, {"env_step": 305000, "rew": -3.5984, "rew_std": 0.1731, "Agent": "SAC"}, {"env_step": 310000, "rew": -4.015, "rew_std": 0.2282, "Agent": "SAC"}, {"env_step": 315000, "rew": -3.95, "rew_std": 0.1787, "Agent": "SAC"}, {"env_step": 320000, "rew": -3.3073, "rew_std": 0.3131, "Agent": "SAC"}, {"env_step": 325000, "rew": -4.1756, "rew_std": 0.3148, "Agent": "SAC"}, {"env_step": 330000, "rew": -3.9191, "rew_std": 0.2974, "Agent": "SAC"}, {"env_step": 335000, "rew": -3.4546, "rew_std": 0.2539, "Agent": "SAC"}, {"env_step": 340000, "rew": -3.4146, "rew_std": 0.2075, "Agent": "SAC"}, {"env_step": 345000, "rew": -3.7292, "rew_std": 0.2684, "Agent": "SAC"}, {"env_step": 350000, "rew": -3.7271, "rew_std": 0.2108, "Agent": "SAC"}, {"env_step": 355000, "rew": -2.995, "rew_std": 0.2411, "Agent": "SAC"}, {"env_step": 360000, "rew": -3.5224, "rew_std": 0.3234, "Agent": "SAC"}, {"env_step": 365000, "rew": -4.4527, "rew_std": 0.1725, "Agent": "SAC"}, {"env_step": 370000, "rew": -3.5645, "rew_std": 0.2728, "Agent": "SAC"}, {"env_step": 375000, "rew": -3.9574, "rew_std": 0.2716, "Agent": "SAC"}, {"env_step": 380000, "rew": -4.0055, "rew_std": 0.2029, "Agent": "SAC"}, {"env_step": 385000, "rew": -3.7493, "rew_std": 0.3596, "Agent": "SAC"}, {"env_step": 390000, "rew": -4.5829, "rew_std": 0.2336, "Agent": "SAC"}, {"env_step": 395000, "rew": -3.7456, "rew_std": 0.3923, "Agent": "SAC"}, {"env_step": 400000, "rew": -3.6572, "rew_std": 0.1587, "Agent": "SAC"}, {"env_step": 405000, "rew": -4.0184, "rew_std": 0.2194, "Agent": "SAC"}, {"env_step": 410000, "rew": -3.9406, "rew_std": 0.1733, "Agent": "SAC"}, {"env_step": 415000, "rew": -3.7399, "rew_std": 0.1998, "Agent": "SAC"}, {"env_step": 420000, "rew": -3.9239, "rew_std": 0.2568, "Agent": "SAC"}, {"env_step": 425000, "rew": -4.1145, "rew_std": 0.1507, "Agent": "SAC"}, {"env_step": 430000, "rew": -3.7682, "rew_std": 0.173, "Agent": "SAC"}, {"env_step": 435000, "rew": -4.1476, "rew_std": 0.1799, "Agent": "SAC"}, {"env_step": 440000, "rew": -3.9612, "rew_std": 0.3424, "Agent": "SAC"}, {"env_step": 445000, "rew": -3.6732, "rew_std": 0.2416, "Agent": "SAC"}, {"env_step": 450000, "rew": -3.7399, "rew_std": 0.3562, "Agent": "SAC"}, {"env_step": 455000, "rew": -3.2862, "rew_std": 0.2323, "Agent": "SAC"}, {"env_step": 460000, "rew": -3.7226, "rew_std": 0.163, "Agent": "SAC"}, {"env_step": 465000, "rew": -3.9655, "rew_std": 0.299, "Agent": "SAC"}, {"env_step": 470000, "rew": -3.5547, "rew_std": 0.2374, "Agent": "SAC"}, {"env_step": 475000, "rew": -3.2005, "rew_std": 0.4062, "Agent": "SAC"}, {"env_step": 480000, "rew": -3.9225, "rew_std": 0.2044, "Agent": "SAC"}, {"env_step": 485000, "rew": -3.9684, "rew_std": 0.2615, "Agent": "SAC"}, {"env_step": 490000, "rew": -3.2122, "rew_std": 0.3031, "Agent": "SAC"}, {"env_step": 495000, "rew": -3.853, "rew_std": 0.1553, "Agent": "SAC"}, {"env_step": 500000, "rew": -3.7073, "rew_std": 0.2017, "Agent": "SAC"}, {"env_step": 505000, "rew": -4.1796, "rew_std": 0.4032, "Agent": "SAC"}, {"env_step": 510000, "rew": -3.6917, "rew_std": 0.2682, "Agent": "SAC"}, {"env_step": 515000, "rew": -3.8461, "rew_std": 0.2467, "Agent": "SAC"}, {"env_step": 520000, "rew": -3.9398, "rew_std": 0.2085, "Agent": "SAC"}, {"env_step": 525000, "rew": -3.7744, "rew_std": 0.2113, "Agent": "SAC"}, {"env_step": 530000, "rew": -3.2922, "rew_std": 0.0859, "Agent": "SAC"}, {"env_step": 535000, "rew": -4.3306, "rew_std": 0.2415, "Agent": "SAC"}, {"env_step": 540000, "rew": -3.5895, "rew_std": 0.2116, "Agent": "SAC"}, {"env_step": 545000, "rew": -3.4097, "rew_std": 0.2857, "Agent": "SAC"}, {"env_step": 550000, "rew": -4.1661, "rew_std": 0.4291, "Agent": "SAC"}, {"env_step": 555000, "rew": -3.8228, "rew_std": 0.261, "Agent": "SAC"}, {"env_step": 560000, "rew": -3.3382, "rew_std": 0.1074, "Agent": "SAC"}, {"env_step": 565000, "rew": -4.3171, "rew_std": 0.3027, "Agent": "SAC"}, {"env_step": 570000, "rew": -3.3844, "rew_std": 0.1957, "Agent": "SAC"}, {"env_step": 575000, "rew": -4.0007, "rew_std": 0.2696, "Agent": "SAC"}, {"env_step": 580000, "rew": -3.264, "rew_std": 0.2616, "Agent": "SAC"}, {"env_step": 585000, "rew": -3.1616, "rew_std": 0.2267, "Agent": "SAC"}, {"env_step": 590000, "rew": -3.2486, "rew_std": 0.1327, "Agent": "SAC"}, {"env_step": 595000, "rew": -3.2078, "rew_std": 0.3255, "Agent": "SAC"}, {"env_step": 600000, "rew": -3.9192, "rew_std": 0.2526, "Agent": "SAC"}, {"env_step": 605000, "rew": -3.8578, "rew_std": 0.1864, "Agent": "SAC"}, {"env_step": 610000, "rew": -4.6848, "rew_std": 0.284, "Agent": "SAC"}, {"env_step": 615000, "rew": -3.9151, "rew_std": 0.2585, "Agent": "SAC"}, {"env_step": 620000, "rew": -4.4256, "rew_std": 0.094, "Agent": "SAC"}, {"env_step": 625000, "rew": -3.5401, "rew_std": 0.4503, "Agent": "SAC"}, {"env_step": 630000, "rew": -3.524, "rew_std": 0.4818, "Agent": "SAC"}, {"env_step": 635000, "rew": -3.442, "rew_std": 0.1696, "Agent": "SAC"}, {"env_step": 640000, "rew": -3.5255, "rew_std": 0.2712, "Agent": "SAC"}, {"env_step": 645000, "rew": -3.782, "rew_std": 0.2156, "Agent": "SAC"}, {"env_step": 650000, "rew": -3.1343, "rew_std": 0.3479, "Agent": "SAC"}, {"env_step": 655000, "rew": -3.8279, "rew_std": 0.2866, "Agent": "SAC"}, {"env_step": 660000, "rew": -4.1674, "rew_std": 0.1903, "Agent": "SAC"}, {"env_step": 665000, "rew": -3.9484, "rew_std": 0.1277, "Agent": "SAC"}, {"env_step": 670000, "rew": -3.5213, "rew_std": 0.3465, "Agent": "SAC"}, {"env_step": 675000, "rew": -3.7349, "rew_std": 0.1057, "Agent": "SAC"}, {"env_step": 680000, "rew": -3.3612, "rew_std": 0.3636, "Agent": "SAC"}, {"env_step": 685000, "rew": -3.8076, "rew_std": 0.3608, "Agent": "SAC"}, {"env_step": 690000, "rew": -3.1594, "rew_std": 0.2127, "Agent": "SAC"}, {"env_step": 695000, "rew": -3.5717, "rew_std": 0.2468, "Agent": "SAC"}, {"env_step": 700000, "rew": -4.2846, "rew_std": 0.3269, "Agent": "SAC"}, {"env_step": 705000, "rew": -2.6392, "rew_std": 0.1967, "Agent": "SAC"}, {"env_step": 710000, "rew": -3.9591, "rew_std": 0.1954, "Agent": "SAC"}, {"env_step": 715000, "rew": -3.8285, "rew_std": 0.1842, "Agent": "SAC"}, {"env_step": 720000, "rew": -4.1874, "rew_std": 0.2372, "Agent": "SAC"}, {"env_step": 725000, "rew": -4.23, "rew_std": 0.2298, "Agent": "SAC"}, {"env_step": 730000, "rew": -3.9943, "rew_std": 0.1125, "Agent": "SAC"}, {"env_step": 735000, "rew": -4.2428, "rew_std": 0.382, "Agent": "SAC"}, {"env_step": 740000, "rew": -3.284, "rew_std": 0.1734, "Agent": "SAC"}, {"env_step": 745000, "rew": -4.3527, "rew_std": 0.3046, "Agent": "SAC"}, {"env_step": 750000, "rew": -3.7276, "rew_std": 0.2788, "Agent": "SAC"}, {"env_step": 755000, "rew": -3.853, "rew_std": 0.2585, "Agent": "SAC"}, {"env_step": 760000, "rew": -4.1638, "rew_std": 0.1737, "Agent": "SAC"}, {"env_step": 765000, "rew": -3.8378, "rew_std": 0.3044, "Agent": "SAC"}, {"env_step": 770000, "rew": -3.6317, "rew_std": 0.1868, "Agent": "SAC"}, {"env_step": 775000, "rew": -3.1226, "rew_std": 0.3625, "Agent": "SAC"}, {"env_step": 780000, "rew": -3.2461, "rew_std": 0.1901, "Agent": "SAC"}, {"env_step": 785000, "rew": -3.7067, "rew_std": 0.2888, "Agent": "SAC"}, {"env_step": 790000, "rew": -3.6783, "rew_std": 0.1692, "Agent": "SAC"}, {"env_step": 795000, "rew": -4.1268, "rew_std": 0.2453, "Agent": "SAC"}, {"env_step": 800000, "rew": -4.0306, "rew_std": 0.1147, "Agent": "SAC"}, {"env_step": 805000, "rew": -3.8568, "rew_std": 0.1906, "Agent": "SAC"}, {"env_step": 810000, "rew": -3.9614, "rew_std": 0.1425, "Agent": "SAC"}, {"env_step": 815000, "rew": -3.5216, "rew_std": 0.2319, "Agent": "SAC"}, {"env_step": 820000, "rew": -4.1455, "rew_std": 0.1742, "Agent": "SAC"}, {"env_step": 825000, "rew": -3.7852, "rew_std": 0.2343, "Agent": "SAC"}, {"env_step": 830000, "rew": -3.8092, "rew_std": 0.3493, "Agent": "SAC"}, {"env_step": 835000, "rew": -3.9306, "rew_std": 0.2396, "Agent": "SAC"}, {"env_step": 840000, "rew": -3.3459, "rew_std": 0.1771, "Agent": "SAC"}, {"env_step": 845000, "rew": -3.8375, "rew_std": 0.2289, "Agent": "SAC"}, {"env_step": 850000, "rew": -3.773, "rew_std": 0.334, "Agent": "SAC"}, {"env_step": 855000, "rew": -3.7423, "rew_std": 0.1455, "Agent": "SAC"}, {"env_step": 860000, "rew": -3.5717, "rew_std": 0.2908, "Agent": "SAC"}, {"env_step": 865000, "rew": -3.793, "rew_std": 0.2614, "Agent": "SAC"}, {"env_step": 870000, "rew": -3.9901, "rew_std": 0.1795, "Agent": "SAC"}, {"env_step": 875000, "rew": -3.3035, "rew_std": 0.1585, "Agent": "SAC"}, {"env_step": 880000, "rew": -3.9038, "rew_std": 0.1819, "Agent": "SAC"}, {"env_step": 885000, "rew": -3.6178, "rew_std": 0.1359, "Agent": "SAC"}, {"env_step": 890000, "rew": -3.048, "rew_std": 0.3071, "Agent": "SAC"}, {"env_step": 895000, "rew": -3.4024, "rew_std": 0.3728, "Agent": "SAC"}, {"env_step": 900000, "rew": -3.0424, "rew_std": 0.1856, "Agent": "SAC"}, {"env_step": 905000, "rew": -4.4867, "rew_std": 0.2158, "Agent": "SAC"}, {"env_step": 910000, "rew": -3.9053, "rew_std": 0.3731, "Agent": "SAC"}, {"env_step": 915000, "rew": -3.8313, "rew_std": 0.2227, "Agent": "SAC"}, {"env_step": 920000, "rew": -4.0479, "rew_std": 0.3955, "Agent": "SAC"}, {"env_step": 925000, "rew": -3.2882, "rew_std": 0.2191, "Agent": "SAC"}, {"env_step": 930000, "rew": -3.6986, "rew_std": 0.3256, "Agent": "SAC"}, {"env_step": 935000, "rew": -4.0628, "rew_std": 0.3019, "Agent": "SAC"}, {"env_step": 940000, "rew": -3.9745, "rew_std": 0.2697, "Agent": "SAC"}, {"env_step": 945000, "rew": -3.6575, "rew_std": 0.2834, "Agent": "SAC"}, {"env_step": 950000, "rew": -3.6307, "rew_std": 0.3706, "Agent": "SAC"}, {"env_step": 955000, "rew": -3.4208, "rew_std": 0.1648, "Agent": "SAC"}, {"env_step": 960000, "rew": -3.3755, "rew_std": 0.286, "Agent": "SAC"}, {"env_step": 965000, "rew": -3.663, "rew_std": 0.2868, "Agent": "SAC"}, {"env_step": 970000, "rew": -4.1586, "rew_std": 0.1768, "Agent": "SAC"}, {"env_step": 975000, "rew": -4.092, "rew_std": 0.1419, "Agent": "SAC"}, {"env_step": 980000, "rew": -4.132, "rew_std": 0.3045, "Agent": "SAC"}, {"env_step": 985000, "rew": -3.7042, "rew_std": 0.3782, "Agent": "SAC"}, {"env_step": 990000, "rew": -2.9588, "rew_std": 0.2477, "Agent": "SAC"}, {"env_step": 995000, "rew": -3.1927, "rew_std": 0.1776, "Agent": "SAC"}, {"env_step": 1000000, "rew": -3.8197, "rew_std": 0.2444, "Agent": "SAC"}, {"env_step": 1005000, "rew": -4.0601, "rew_std": 0.1803, "Agent": "SAC"}, {"env_step": 1010000, "rew": -2.9736, "rew_std": 0.1404, "Agent": "SAC"}, {"env_step": 1015000, "rew": -4.093, "rew_std": 0.327, "Agent": "SAC"}, {"env_step": 1020000, "rew": -3.9777, "rew_std": 0.3032, "Agent": "SAC"}, {"env_step": 1025000, "rew": -4.2207, "rew_std": 0.1991, "Agent": "SAC"}, {"env_step": 1030000, "rew": -4.2962, "rew_std": 0.2042, "Agent": "SAC"}, {"env_step": 1035000, "rew": -3.8344, "rew_std": 0.1368, "Agent": "SAC"}, {"env_step": 1040000, "rew": -4.1522, "rew_std": 0.2222, "Agent": "SAC"}, {"env_step": 1045000, "rew": -3.6833, "rew_std": 0.2655, "Agent": "SAC"}, {"env_step": 1050000, "rew": -3.1982, "rew_std": 0.2894, "Agent": "SAC"}, {"env_step": 1055000, "rew": -3.9271, "rew_std": 0.4604, "Agent": "SAC"}, {"env_step": 1060000, "rew": -3.7066, "rew_std": 0.2758, "Agent": "SAC"}, {"env_step": 1065000, "rew": -4.0772, "rew_std": 0.1879, "Agent": "SAC"}, {"env_step": 1070000, "rew": -3.707, "rew_std": 0.1503, "Agent": "SAC"}, {"env_step": 1075000, "rew": -2.8186, "rew_std": 0.3516, "Agent": "SAC"}, {"env_step": 1080000, "rew": -3.5769, "rew_std": 0.0814, "Agent": "SAC"}, {"env_step": 1085000, "rew": -3.9369, "rew_std": 0.1667, "Agent": "SAC"}, {"env_step": 1090000, "rew": -3.199, "rew_std": 0.2651, "Agent": "SAC"}, {"env_step": 1095000, "rew": -3.452, "rew_std": 0.268, "Agent": "SAC"}, {"env_step": 1100000, "rew": -4.5154, "rew_std": 0.303, "Agent": "SAC"}, {"env_step": 1105000, "rew": -3.4344, "rew_std": 0.1963, "Agent": "SAC"}, {"env_step": 1110000, "rew": -4.3523, "rew_std": 0.2202, "Agent": "SAC"}, {"env_step": 1115000, "rew": -4.0188, "rew_std": 0.2337, "Agent": "SAC"}, {"env_step": 1120000, "rew": -4.0969, "rew_std": 0.478, "Agent": "SAC"}, {"env_step": 1125000, "rew": -3.6457, "rew_std": 0.2369, "Agent": "SAC"}, {"env_step": 1130000, "rew": -4.2586, "rew_std": 0.3208, "Agent": "SAC"}, {"env_step": 1135000, "rew": -3.4311, "rew_std": 0.1391, "Agent": "SAC"}, {"env_step": 1140000, "rew": -3.7406, "rew_std": 0.2248, "Agent": "SAC"}, {"env_step": 1145000, "rew": -3.6664, "rew_std": 0.152, "Agent": "SAC"}, {"env_step": 1150000, "rew": -3.6026, "rew_std": 0.2842, "Agent": "SAC"}, {"env_step": 1155000, "rew": -3.9512, "rew_std": 0.1563, "Agent": "SAC"}, {"env_step": 1160000, "rew": -3.2359, "rew_std": 0.2899, "Agent": "SAC"}, {"env_step": 1165000, "rew": -4.352, "rew_std": 0.4036, "Agent": "SAC"}, {"env_step": 1170000, "rew": -4.4763, "rew_std": 0.4422, "Agent": "SAC"}, {"env_step": 1175000, "rew": -2.887, "rew_std": 0.1978, "Agent": "SAC"}, {"env_step": 1180000, "rew": -3.366, "rew_std": 0.1115, "Agent": "SAC"}, {"env_step": 1185000, "rew": -3.7818, "rew_std": 0.3499, "Agent": "SAC"}, {"env_step": 1190000, "rew": -4.2838, "rew_std": 0.2112, "Agent": "SAC"}, {"env_step": 1195000, "rew": -3.8644, "rew_std": 0.1579, "Agent": "SAC"}, {"env_step": 1200000, "rew": -3.9126, "rew_std": 0.3867, "Agent": "SAC"}, {"env_step": 1205000, "rew": -3.6113, "rew_std": 0.1608, "Agent": "SAC"}, {"env_step": 1210000, "rew": -3.6634, "rew_std": 0.3782, "Agent": "SAC"}, {"env_step": 1215000, "rew": -3.469, "rew_std": 0.3198, "Agent": "SAC"}, {"env_step": 1220000, "rew": -3.9637, "rew_std": 0.3019, "Agent": "SAC"}, {"env_step": 1225000, "rew": -3.3988, "rew_std": 0.1586, "Agent": "SAC"}, {"env_step": 1230000, "rew": -3.798, "rew_std": 0.157, "Agent": "SAC"}, {"env_step": 1235000, "rew": -4.1499, "rew_std": 0.111, "Agent": "SAC"}, {"env_step": 1240000, "rew": -3.4294, "rew_std": 0.2204, "Agent": "SAC"}, {"env_step": 1245000, "rew": -3.9595, "rew_std": 0.1531, "Agent": "SAC"}, {"env_step": 1250000, "rew": -4.1026, "rew_std": 0.2315, "Agent": "SAC"}, {"env_step": 0, "rew": -27.3232, "rew_std": 23.4601, "Agent": "DDPG"}, {"env_step": 5000, "rew": -5.9994, "rew_std": 0.4863, "Agent": "DDPG"}, {"env_step": 10000, "rew": -5.9481, "rew_std": 0.5947, "Agent": "DDPG"}, {"env_step": 15000, "rew": -6.5493, "rew_std": 0.871, "Agent": "DDPG"}, {"env_step": 20000, "rew": -5.3363, "rew_std": 0.6571, "Agent": "DDPG"}, {"env_step": 25000, "rew": -6.3717, "rew_std": 0.7762, "Agent": "DDPG"}, {"env_step": 30000, "rew": -5.3757, "rew_std": 0.4863, "Agent": "DDPG"}, {"env_step": 35000, "rew": -5.3225, "rew_std": 0.3755, "Agent": "DDPG"}, {"env_step": 40000, "rew": -5.1545, "rew_std": 0.2959, "Agent": "DDPG"}, {"env_step": 45000, "rew": -5.1735, "rew_std": 0.5609, "Agent": "DDPG"}, {"env_step": 50000, "rew": -5.0773, "rew_std": 0.3339, "Agent": "DDPG"}, {"env_step": 55000, "rew": -5.1035, "rew_std": 0.3217, "Agent": "DDPG"}, {"env_step": 60000, "rew": -4.4369, "rew_std": 0.3355, "Agent": "DDPG"}, {"env_step": 65000, "rew": -4.1462, "rew_std": 0.2317, "Agent": "DDPG"}, {"env_step": 70000, "rew": -4.7954, "rew_std": 0.4211, "Agent": "DDPG"}, {"env_step": 75000, "rew": -4.3913, "rew_std": 0.1985, "Agent": "DDPG"}, {"env_step": 80000, "rew": -4.5597, "rew_std": 0.2783, "Agent": "DDPG"}, {"env_step": 85000, "rew": -4.3348, "rew_std": 0.2622, "Agent": "DDPG"}, {"env_step": 90000, "rew": -5.0381, "rew_std": 0.4593, "Agent": "DDPG"}, {"env_step": 95000, "rew": -4.6645, "rew_std": 0.3458, "Agent": "DDPG"}, {"env_step": 100000, "rew": -4.5549, "rew_std": 0.188, "Agent": "DDPG"}, {"env_step": 105000, "rew": -4.6647, "rew_std": 0.182, "Agent": "DDPG"}, {"env_step": 110000, "rew": -4.8689, "rew_std": 0.2584, "Agent": "DDPG"}, {"env_step": 115000, "rew": -4.1283, "rew_std": 0.1862, "Agent": "DDPG"}, {"env_step": 120000, "rew": -3.6985, "rew_std": 0.2483, "Agent": "DDPG"}, {"env_step": 125000, "rew": -3.8125, "rew_std": 0.2723, "Agent": "DDPG"}, {"env_step": 130000, "rew": -4.3951, "rew_std": 0.2996, "Agent": "DDPG"}, {"env_step": 135000, "rew": -4.2906, "rew_std": 0.4147, "Agent": "DDPG"}, {"env_step": 140000, "rew": -4.4004, "rew_std": 0.3321, "Agent": "DDPG"}, {"env_step": 145000, "rew": -4.9837, "rew_std": 0.3403, "Agent": "DDPG"}, {"env_step": 150000, "rew": -4.4368, "rew_std": 0.2152, "Agent": "DDPG"}, {"env_step": 155000, "rew": -4.2693, "rew_std": 0.4226, "Agent": "DDPG"}, {"env_step": 160000, "rew": -4.2912, "rew_std": 0.2346, "Agent": "DDPG"}, {"env_step": 165000, "rew": -4.357, "rew_std": 0.3799, "Agent": "DDPG"}, {"env_step": 170000, "rew": -4.3997, "rew_std": 0.2363, "Agent": "DDPG"}, {"env_step": 175000, "rew": -4.3367, "rew_std": 0.1632, "Agent": "DDPG"}, {"env_step": 180000, "rew": -4.3578, "rew_std": 0.1974, "Agent": "DDPG"}, {"env_step": 185000, "rew": -3.8391, "rew_std": 0.2901, "Agent": "DDPG"}, {"env_step": 190000, "rew": -4.132, "rew_std": 0.2211, "Agent": "DDPG"}, {"env_step": 195000, "rew": -3.8273, "rew_std": 0.2488, "Agent": "DDPG"}, {"env_step": 200000, "rew": -3.7421, "rew_std": 0.2776, "Agent": "DDPG"}, {"env_step": 205000, "rew": -4.2475, "rew_std": 0.2108, "Agent": "DDPG"}, {"env_step": 210000, "rew": -4.4562, "rew_std": 0.1796, "Agent": "DDPG"}, {"env_step": 215000, "rew": -4.5319, "rew_std": 0.2057, "Agent": "DDPG"}, {"env_step": 220000, "rew": -3.9499, "rew_std": 0.3337, "Agent": "DDPG"}, {"env_step": 225000, "rew": -4.3857, "rew_std": 0.1958, "Agent": "DDPG"}, {"env_step": 230000, "rew": -4.2473, "rew_std": 0.1687, "Agent": "DDPG"}, {"env_step": 235000, "rew": -3.4639, "rew_std": 0.26, "Agent": "DDPG"}, {"env_step": 240000, "rew": -4.5433, "rew_std": 0.2626, "Agent": "DDPG"}, {"env_step": 245000, "rew": -4.3408, "rew_std": 0.2718, "Agent": "DDPG"}, {"env_step": 250000, "rew": -4.1629, "rew_std": 0.2863, "Agent": "DDPG"}, {"env_step": 255000, "rew": -3.6422, "rew_std": 0.3542, "Agent": "DDPG"}, {"env_step": 260000, "rew": -4.1054, "rew_std": 0.1149, "Agent": "DDPG"}, {"env_step": 265000, "rew": -3.9174, "rew_std": 0.2286, "Agent": "DDPG"}, {"env_step": 270000, "rew": -4.0504, "rew_std": 0.2778, "Agent": "DDPG"}, {"env_step": 275000, "rew": -4.0093, "rew_std": 0.24, "Agent": "DDPG"}, {"env_step": 280000, "rew": -4.1029, "rew_std": 0.2496, "Agent": "DDPG"}, {"env_step": 285000, "rew": -4.492, "rew_std": 0.2369, "Agent": "DDPG"}, {"env_step": 290000, "rew": -3.8098, "rew_std": 0.1355, "Agent": "DDPG"}, {"env_step": 295000, "rew": -3.905, "rew_std": 0.1612, "Agent": "DDPG"}, {"env_step": 300000, "rew": -4.1215, "rew_std": 0.5278, "Agent": "DDPG"}, {"env_step": 305000, "rew": -4.007, "rew_std": 0.1152, "Agent": "DDPG"}, {"env_step": 310000, "rew": -4.3045, "rew_std": 0.1947, "Agent": "DDPG"}, {"env_step": 315000, "rew": -4.2704, "rew_std": 0.1864, "Agent": "DDPG"}, {"env_step": 320000, "rew": -3.6252, "rew_std": 0.27, "Agent": "DDPG"}, {"env_step": 325000, "rew": -4.5046, "rew_std": 0.3124, "Agent": "DDPG"}, {"env_step": 330000, "rew": -4.3069, "rew_std": 0.2592, "Agent": "DDPG"}, {"env_step": 335000, "rew": -3.7179, "rew_std": 0.3197, "Agent": "DDPG"}, {"env_step": 340000, "rew": -3.7661, "rew_std": 0.2643, "Agent": "DDPG"}, {"env_step": 345000, "rew": -4.0593, "rew_std": 0.2877, "Agent": "DDPG"}, {"env_step": 350000, "rew": -4.0207, "rew_std": 0.2279, "Agent": "DDPG"}, {"env_step": 355000, "rew": -3.2942, "rew_std": 0.2682, "Agent": "DDPG"}, {"env_step": 360000, "rew": -3.8581, "rew_std": 0.3476, "Agent": "DDPG"}, {"env_step": 365000, "rew": -4.7662, "rew_std": 0.2355, "Agent": "DDPG"}, {"env_step": 370000, "rew": -3.8946, "rew_std": 0.3209, "Agent": "DDPG"}, {"env_step": 375000, "rew": -4.2492, "rew_std": 0.2504, "Agent": "DDPG"}, {"env_step": 380000, "rew": -4.3849, "rew_std": 0.2134, "Agent": "DDPG"}, {"env_step": 385000, "rew": -4.0957, "rew_std": 0.4477, "Agent": "DDPG"}, {"env_step": 390000, "rew": -4.9408, "rew_std": 0.1723, "Agent": "DDPG"}, {"env_step": 395000, "rew": -4.0376, "rew_std": 0.3935, "Agent": "DDPG"}, {"env_step": 400000, "rew": -4.0078, "rew_std": 0.1518, "Agent": "DDPG"}, {"env_step": 405000, "rew": -4.3918, "rew_std": 0.301, "Agent": "DDPG"}, {"env_step": 410000, "rew": -4.2993, "rew_std": 0.1881, "Agent": "DDPG"}, {"env_step": 415000, "rew": -4.065, "rew_std": 0.2356, "Agent": "DDPG"}, {"env_step": 420000, "rew": -4.284, "rew_std": 0.2906, "Agent": "DDPG"}, {"env_step": 425000, "rew": -4.431, "rew_std": 0.1105, "Agent": "DDPG"}, {"env_step": 430000, "rew": -4.2572, "rew_std": 0.3112, "Agent": "DDPG"}, {"env_step": 435000, "rew": -4.4992, "rew_std": 0.2926, "Agent": "DDPG"}, {"env_step": 440000, "rew": -4.4105, "rew_std": 0.3677, "Agent": "DDPG"}, {"env_step": 445000, "rew": -4.1118, "rew_std": 0.3003, "Agent": "DDPG"}, {"env_step": 450000, "rew": -4.1832, "rew_std": 0.3488, "Agent": "DDPG"}, {"env_step": 455000, "rew": -3.6541, "rew_std": 0.3019, "Agent": "DDPG"}, {"env_step": 460000, "rew": -4.1346, "rew_std": 0.192, "Agent": "DDPG"}, {"env_step": 465000, "rew": -4.4254, "rew_std": 0.3568, "Agent": "DDPG"}, {"env_step": 470000, "rew": -3.9608, "rew_std": 0.2456, "Agent": "DDPG"}, {"env_step": 475000, "rew": -3.8078, "rew_std": 0.6135, "Agent": "DDPG"}, {"env_step": 480000, "rew": -4.3577, "rew_std": 0.2294, "Agent": "DDPG"}, {"env_step": 485000, "rew": -4.2809, "rew_std": 0.2382, "Agent": "DDPG"}, {"env_step": 490000, "rew": -3.5478, "rew_std": 0.3021, "Agent": "DDPG"}, {"env_step": 495000, "rew": -4.309, "rew_std": 0.419, "Agent": "DDPG"}, {"env_step": 500000, "rew": -4.184, "rew_std": 0.1899, "Agent": "DDPG"}, {"env_step": 505000, "rew": -4.6083, "rew_std": 0.4478, "Agent": "DDPG"}, {"env_step": 510000, "rew": -4.193, "rew_std": 0.4263, "Agent": "DDPG"}, {"env_step": 515000, "rew": -4.4484, "rew_std": 0.6183, "Agent": "DDPG"}, {"env_step": 520000, "rew": -4.5897, "rew_std": 0.3247, "Agent": "DDPG"}, {"env_step": 525000, "rew": -4.3144, "rew_std": 0.4962, "Agent": "DDPG"}, {"env_step": 530000, "rew": -3.7567, "rew_std": 0.3392, "Agent": "DDPG"}, {"env_step": 535000, "rew": -4.8428, "rew_std": 0.2237, "Agent": "DDPG"}, {"env_step": 540000, "rew": -4.0561, "rew_std": 0.3695, "Agent": "DDPG"}, {"env_step": 545000, "rew": -3.7848, "rew_std": 0.3187, "Agent": "DDPG"}, {"env_step": 550000, "rew": -4.7356, "rew_std": 0.3935, "Agent": "DDPG"}, {"env_step": 555000, "rew": -4.3297, "rew_std": 0.2376, "Agent": "DDPG"}, {"env_step": 560000, "rew": -3.7468, "rew_std": 0.132, "Agent": "DDPG"}, {"env_step": 565000, "rew": -4.7899, "rew_std": 0.2909, "Agent": "DDPG"}, {"env_step": 570000, "rew": -3.826, "rew_std": 0.2805, "Agent": "DDPG"}, {"env_step": 575000, "rew": -4.3571, "rew_std": 0.3701, "Agent": "DDPG"}, {"env_step": 580000, "rew": -3.8213, "rew_std": 0.2697, "Agent": "DDPG"}, {"env_step": 585000, "rew": -3.8807, "rew_std": 0.577, "Agent": "DDPG"}, {"env_step": 590000, "rew": -3.7687, "rew_std": 0.2783, "Agent": "DDPG"}, {"env_step": 595000, "rew": -3.8111, "rew_std": 0.2123, "Agent": "DDPG"}, {"env_step": 600000, "rew": -4.4374, "rew_std": 0.2168, "Agent": "DDPG"}, {"env_step": 605000, "rew": -4.4213, "rew_std": 0.3345, "Agent": "DDPG"}, {"env_step": 610000, "rew": -5.3496, "rew_std": 0.1879, "Agent": "DDPG"}, {"env_step": 615000, "rew": -4.6118, "rew_std": 0.4872, "Agent": "DDPG"}, {"env_step": 620000, "rew": -5.0175, "rew_std": 0.3868, "Agent": "DDPG"}, {"env_step": 625000, "rew": -4.2684, "rew_std": 0.4877, "Agent": "DDPG"}, {"env_step": 630000, "rew": -4.2208, "rew_std": 0.5298, "Agent": "DDPG"}, {"env_step": 635000, "rew": -4.0813, "rew_std": 0.341, "Agent": "DDPG"}, {"env_step": 640000, "rew": -4.1259, "rew_std": 0.2516, "Agent": "DDPG"}, {"env_step": 645000, "rew": -4.3379, "rew_std": 0.1992, "Agent": "DDPG"}, {"env_step": 650000, "rew": -3.7851, "rew_std": 0.3134, "Agent": "DDPG"}, {"env_step": 655000, "rew": -4.5744, "rew_std": 0.4368, "Agent": "DDPG"}, {"env_step": 660000, "rew": -4.7546, "rew_std": 0.121, "Agent": "DDPG"}, {"env_step": 665000, "rew": -4.6352, "rew_std": 0.3387, "Agent": "DDPG"}, {"env_step": 670000, "rew": -4.0116, "rew_std": 0.4186, "Agent": "DDPG"}, {"env_step": 675000, "rew": -4.2434, "rew_std": 0.2248, "Agent": "DDPG"}, {"env_step": 680000, "rew": -4.0583, "rew_std": 0.4662, "Agent": "DDPG"}, {"env_step": 685000, "rew": -4.6387, "rew_std": 0.6938, "Agent": "DDPG"}, {"env_step": 690000, "rew": -3.9708, "rew_std": 0.345, "Agent": "DDPG"}, {"env_step": 695000, "rew": -4.3476, "rew_std": 0.3352, "Agent": "DDPG"}, {"env_step": 700000, "rew": -4.8799, "rew_std": 0.2911, "Agent": "DDPG"}, {"env_step": 705000, "rew": -3.6108, "rew_std": 0.826, "Agent": "DDPG"}, {"env_step": 710000, "rew": -4.6422, "rew_std": 0.2948, "Agent": "DDPG"}, {"env_step": 715000, "rew": -4.4962, "rew_std": 0.3051, "Agent": "DDPG"}, {"env_step": 720000, "rew": -4.9724, "rew_std": 0.3331, "Agent": "DDPG"}, {"env_step": 725000, "rew": -4.9428, "rew_std": 0.402, "Agent": "DDPG"}, {"env_step": 730000, "rew": -4.8311, "rew_std": 0.3027, "Agent": "DDPG"}, {"env_step": 735000, "rew": -4.9349, "rew_std": 0.4743, "Agent": "DDPG"}, {"env_step": 740000, "rew": -4.1416, "rew_std": 0.3314, "Agent": "DDPG"}, {"env_step": 745000, "rew": -5.0792, "rew_std": 0.396, "Agent": "DDPG"}, {"env_step": 750000, "rew": -4.3809, "rew_std": 0.3205, "Agent": "DDPG"}, {"env_step": 755000, "rew": -4.7253, "rew_std": 0.3842, "Agent": "DDPG"}, {"env_step": 760000, "rew": -4.9929, "rew_std": 0.4151, "Agent": "DDPG"}, {"env_step": 765000, "rew": -4.7099, "rew_std": 0.5626, "Agent": "DDPG"}, {"env_step": 770000, "rew": -4.4012, "rew_std": 0.3591, "Agent": "DDPG"}, {"env_step": 775000, "rew": -3.8417, "rew_std": 0.3934, "Agent": "DDPG"}, {"env_step": 780000, "rew": -3.9774, "rew_std": 0.2247, "Agent": "DDPG"}, {"env_step": 785000, "rew": -4.5148, "rew_std": 0.4076, "Agent": "DDPG"}, {"env_step": 790000, "rew": -4.4641, "rew_std": 0.244, "Agent": "DDPG"}, {"env_step": 795000, "rew": -5.0113, "rew_std": 0.4511, "Agent": "DDPG"}, {"env_step": 800000, "rew": -4.7334, "rew_std": 0.2928, "Agent": "DDPG"}, {"env_step": 805000, "rew": -4.5561, "rew_std": 0.2719, "Agent": "DDPG"}, {"env_step": 810000, "rew": -4.6724, "rew_std": 0.2335, "Agent": "DDPG"}, {"env_step": 815000, "rew": -4.3218, "rew_std": 0.2871, "Agent": "DDPG"}, {"env_step": 820000, "rew": -5.0469, "rew_std": 0.4414, "Agent": "DDPG"}, {"env_step": 825000, "rew": -4.653, "rew_std": 0.4338, "Agent": "DDPG"}, {"env_step": 830000, "rew": -4.8143, "rew_std": 0.525, "Agent": "DDPG"}, {"env_step": 835000, "rew": -4.8063, "rew_std": 0.3875, "Agent": "DDPG"}, {"env_step": 840000, "rew": -4.106, "rew_std": 0.2461, "Agent": "DDPG"}, {"env_step": 845000, "rew": -4.5475, "rew_std": 0.2879, "Agent": "DDPG"}, {"env_step": 850000, "rew": -4.7167, "rew_std": 0.5187, "Agent": "DDPG"}, {"env_step": 855000, "rew": -4.6854, "rew_std": 0.2957, "Agent": "DDPG"}, {"env_step": 860000, "rew": -4.3747, "rew_std": 0.2721, "Agent": "DDPG"}, {"env_step": 865000, "rew": -4.4879, "rew_std": 0.3751, "Agent": "DDPG"}, {"env_step": 870000, "rew": -4.8097, "rew_std": 0.1644, "Agent": "DDPG"}, {"env_step": 875000, "rew": -4.1463, "rew_std": 0.3703, "Agent": "DDPG"}, {"env_step": 880000, "rew": -4.6397, "rew_std": 0.4628, "Agent": "DDPG"}, {"env_step": 885000, "rew": -4.554, "rew_std": 0.2611, "Agent": "DDPG"}, {"env_step": 890000, "rew": -4.0128, "rew_std": 0.3459, "Agent": "DDPG"}, {"env_step": 895000, "rew": -4.3408, "rew_std": 0.5245, "Agent": "DDPG"}, {"env_step": 900000, "rew": -3.8282, "rew_std": 0.2118, "Agent": "DDPG"}, {"env_step": 905000, "rew": -5.5158, "rew_std": 0.5981, "Agent": "DDPG"}, {"env_step": 910000, "rew": -4.8566, "rew_std": 0.4799, "Agent": "DDPG"}, {"env_step": 915000, "rew": -4.8133, "rew_std": 0.3469, "Agent": "DDPG"}, {"env_step": 920000, "rew": -5.049, "rew_std": 0.5046, "Agent": "DDPG"}, {"env_step": 925000, "rew": -4.2925, "rew_std": 0.3937, "Agent": "DDPG"}, {"env_step": 930000, "rew": -4.9669, "rew_std": 0.417, "Agent": "DDPG"}, {"env_step": 935000, "rew": -5.1534, "rew_std": 0.7335, "Agent": "DDPG"}, {"env_step": 940000, "rew": -5.2228, "rew_std": 0.6543, "Agent": "DDPG"}, {"env_step": 945000, "rew": -4.6248, "rew_std": 0.6695, "Agent": "DDPG"}, {"env_step": 950000, "rew": -4.8075, "rew_std": 0.5397, "Agent": "DDPG"}, {"env_step": 955000, "rew": -4.5568, "rew_std": 0.4897, "Agent": "DDPG"}, {"env_step": 960000, "rew": -4.7383, "rew_std": 1.3685, "Agent": "DDPG"}, {"env_step": 965000, "rew": -4.9158, "rew_std": 0.7738, "Agent": "DDPG"}, {"env_step": 970000, "rew": -5.361, "rew_std": 0.4219, "Agent": "DDPG"}, {"env_step": 975000, "rew": -5.0993, "rew_std": 0.5179, "Agent": "DDPG"}, {"env_step": 980000, "rew": -5.2621, "rew_std": 0.5666, "Agent": "DDPG"}, {"env_step": 985000, "rew": -4.7954, "rew_std": 0.6463, "Agent": "DDPG"}, {"env_step": 990000, "rew": -4.0475, "rew_std": 0.4556, "Agent": "DDPG"}, {"env_step": 995000, "rew": -4.261, "rew_std": 0.3856, "Agent": "DDPG"}, {"env_step": 1000000, "rew": -5.1163, "rew_std": 0.4632, "Agent": "DDPG"}, {"env_step": 1005000, "rew": -5.6486, "rew_std": 1.3292, "Agent": "DDPG"}, {"env_step": 1010000, "rew": -5.588, "rew_std": 1.9266, "Agent": "DDPG"}, {"env_step": 1015000, "rew": -7.7173, "rew_std": 3.5321, "Agent": "DDPG"}, {"env_step": 1020000, "rew": -7.9336, "rew_std": 3.2407, "Agent": "DDPG"}, {"env_step": 1025000, "rew": -9.0305, "rew_std": 4.5181, "Agent": "DDPG"}, {"env_step": 1030000, "rew": -11.4767, "rew_std": 8.7641, "Agent": "DDPG"}, {"env_step": 1035000, "rew": -9.7738, "rew_std": 6.5195, "Agent": "DDPG"}, {"env_step": 1040000, "rew": -10.5483, "rew_std": 7.6541, "Agent": "DDPG"}, {"env_step": 1045000, "rew": -13.2258, "rew_std": 15.421, "Agent": "DDPG"}, {"env_step": 1050000, "rew": -20.784, "rew_std": 23.1296, "Agent": "DDPG"}, {"env_step": 1055000, "rew": -18.1494, "rew_std": 20.8153, "Agent": "DDPG"}, {"env_step": 1060000, "rew": -20.7918, "rew_std": 27.238, "Agent": "DDPG"}, {"env_step": 1065000, "rew": -14.8026, "rew_std": 12.7541, "Agent": "DDPG"}, {"env_step": 1070000, "rew": -19.7576, "rew_std": 22.8814, "Agent": "DDPG"}, {"env_step": 1075000, "rew": -15.4066, "rew_std": 12.2726, "Agent": "DDPG"}, {"env_step": 1080000, "rew": -12.5202, "rew_std": 7.5183, "Agent": "DDPG"}, {"env_step": 1085000, "rew": -20.5579, "rew_std": 17.4525, "Agent": "DDPG"}, {"env_step": 1090000, "rew": -14.0937, "rew_std": 11.4289, "Agent": "DDPG"}, {"env_step": 1095000, "rew": -15.7974, "rew_std": 12.7581, "Agent": "DDPG"}, {"env_step": 1100000, "rew": -15.1148, "rew_std": 10.7843, "Agent": "DDPG"}, {"env_step": 1105000, "rew": -16.2613, "rew_std": 12.6272, "Agent": "DDPG"}, {"env_step": 1110000, "rew": -17.2591, "rew_std": 15.419, "Agent": "DDPG"}, {"env_step": 1115000, "rew": -19.0267, "rew_std": 15.5475, "Agent": "DDPG"}, {"env_step": 1120000, "rew": -22.7455, "rew_std": 17.976, "Agent": "DDPG"}, {"env_step": 1125000, "rew": -18.2955, "rew_std": 14.3416, "Agent": "DDPG"}, {"env_step": 1130000, "rew": -20.4421, "rew_std": 11.4797, "Agent": "DDPG"}, {"env_step": 1135000, "rew": -15.9736, "rew_std": 6.5837, "Agent": "DDPG"}, {"env_step": 1140000, "rew": -19.6391, "rew_std": 13.1862, "Agent": "DDPG"}, {"env_step": 1145000, "rew": -23.2974, "rew_std": 13.0105, "Agent": "DDPG"}, {"env_step": 1150000, "rew": -21.9923, "rew_std": 11.1915, "Agent": "DDPG"}, {"env_step": 1155000, "rew": -25.8502, "rew_std": 14.6756, "Agent": "DDPG"}, {"env_step": 1160000, "rew": -26.8855, "rew_std": 18.5184, "Agent": "DDPG"}, {"env_step": 1165000, "rew": -22.6703, "rew_std": 11.9136, "Agent": "DDPG"}, {"env_step": 1170000, "rew": -25.7278, "rew_std": 11.4867, "Agent": "DDPG"}, {"env_step": 1175000, "rew": -18.4098, "rew_std": 8.5134, "Agent": "DDPG"}, {"env_step": 1180000, "rew": -25.0163, "rew_std": 14.2717, "Agent": "DDPG"}, {"env_step": 1185000, "rew": -17.7322, "rew_std": 9.6582, "Agent": "DDPG"}, {"env_step": 1190000, "rew": -20.5272, "rew_std": 12.0385, "Agent": "DDPG"}, {"env_step": 1195000, "rew": -25.2786, "rew_std": 23.9859, "Agent": "DDPG"}, {"env_step": 1200000, "rew": -29.8174, "rew_std": 32.2052, "Agent": "DDPG"}, {"env_step": 1205000, "rew": -22.3033, "rew_std": 18.2211, "Agent": "DDPG"}, {"env_step": 1210000, "rew": -18.0601, "rew_std": 6.9453, "Agent": "DDPG"}, {"env_step": 1215000, "rew": -16.9678, "rew_std": 9.2312, "Agent": "DDPG"}, {"env_step": 1220000, "rew": -15.8163, "rew_std": 10.0092, "Agent": "DDPG"}, {"env_step": 1225000, "rew": -14.2974, "rew_std": 9.1534, "Agent": "DDPG"}, {"env_step": 1230000, "rew": -13.7051, "rew_std": 8.1698, "Agent": "DDPG"}, {"env_step": 1235000, "rew": -12.1851, "rew_std": 7.4597, "Agent": "DDPG"}, {"env_step": 1240000, "rew": -10.4908, "rew_std": 6.1628, "Agent": "DDPG"}, {"env_step": 1245000, "rew": -12.2744, "rew_std": 9.3555, "Agent": "DDPG"}, {"env_step": 1250000, "rew": -10.4825, "rew_std": 4.1534, "Agent": "DDPG"}, {"env_step": 0, "rew": -27.3232, "rew_std": 23.4601, "Agent": "TD3"}, {"env_step": 5000, "rew": -9.4278, "rew_std": 0.6088, "Agent": "TD3"}, {"env_step": 10000, "rew": -7.0669, "rew_std": 0.5544, "Agent": "TD3"}, {"env_step": 15000, "rew": -6.4267, "rew_std": 0.5535, "Agent": "TD3"}, {"env_step": 20000, "rew": -5.1856, "rew_std": 0.5122, "Agent": "TD3"}, {"env_step": 25000, "rew": -6.2298, "rew_std": 0.4873, "Agent": "TD3"}, {"env_step": 30000, "rew": -4.746, "rew_std": 0.2296, "Agent": "TD3"}, {"env_step": 35000, "rew": -4.9355, "rew_std": 0.4494, "Agent": "TD3"}, {"env_step": 40000, "rew": -5.18, "rew_std": 0.4861, "Agent": "TD3"}, {"env_step": 45000, "rew": -5.3703, "rew_std": 0.797, "Agent": "TD3"}, {"env_step": 50000, "rew": -5.2779, "rew_std": 0.5659, "Agent": "TD3"}, {"env_step": 55000, "rew": -5.2297, "rew_std": 0.3341, "Agent": "TD3"}, {"env_step": 60000, "rew": -4.5756, "rew_std": 0.476, "Agent": "TD3"}, {"env_step": 65000, "rew": -4.2738, "rew_std": 0.1586, "Agent": "TD3"}, {"env_step": 70000, "rew": -4.8618, "rew_std": 0.3402, "Agent": "TD3"}, {"env_step": 75000, "rew": -4.2569, "rew_std": 0.1832, "Agent": "TD3"}, {"env_step": 80000, "rew": -4.5128, "rew_std": 0.3919, "Agent": "TD3"}, {"env_step": 85000, "rew": -4.45, "rew_std": 0.1863, "Agent": "TD3"}, {"env_step": 90000, "rew": -5.1534, "rew_std": 0.3301, "Agent": "TD3"}, {"env_step": 95000, "rew": -4.618, "rew_std": 0.218, "Agent": "TD3"}, {"env_step": 100000, "rew": -4.6005, "rew_std": 0.2978, "Agent": "TD3"}, {"env_step": 105000, "rew": -4.7658, "rew_std": 0.3353, "Agent": "TD3"}, {"env_step": 110000, "rew": -4.8709, "rew_std": 0.4209, "Agent": "TD3"}, {"env_step": 115000, "rew": -3.8872, "rew_std": 0.1921, "Agent": "TD3"}, {"env_step": 120000, "rew": -3.5985, "rew_std": 0.2389, "Agent": "TD3"}, {"env_step": 125000, "rew": -3.8332, "rew_std": 0.2889, "Agent": "TD3"}, {"env_step": 130000, "rew": -4.3352, "rew_std": 0.2688, "Agent": "TD3"}, {"env_step": 135000, "rew": -4.2144, "rew_std": 0.5016, "Agent": "TD3"}, {"env_step": 140000, "rew": -4.4658, "rew_std": 0.3705, "Agent": "TD3"}, {"env_step": 145000, "rew": -4.9148, "rew_std": 0.1617, "Agent": "TD3"}, {"env_step": 150000, "rew": -4.37, "rew_std": 0.1933, "Agent": "TD3"}, {"env_step": 155000, "rew": -4.3036, "rew_std": 0.4578, "Agent": "TD3"}, {"env_step": 160000, "rew": -4.2312, "rew_std": 0.2589, "Agent": "TD3"}, {"env_step": 165000, "rew": -4.3598, "rew_std": 0.4077, "Agent": "TD3"}, {"env_step": 170000, "rew": -4.3913, "rew_std": 0.2397, "Agent": "TD3"}, {"env_step": 175000, "rew": -4.3605, "rew_std": 0.1915, "Agent": "TD3"}, {"env_step": 180000, "rew": -4.3441, "rew_std": 0.1972, "Agent": "TD3"}, {"env_step": 185000, "rew": -3.8606, "rew_std": 0.2643, "Agent": "TD3"}, {"env_step": 190000, "rew": -4.1763, "rew_std": 0.1379, "Agent": "TD3"}, {"env_step": 195000, "rew": -3.8127, "rew_std": 0.2737, "Agent": "TD3"}, {"env_step": 200000, "rew": -3.774, "rew_std": 0.3018, "Agent": "TD3"}, {"env_step": 205000, "rew": -4.2305, "rew_std": 0.301, "Agent": "TD3"}, {"env_step": 210000, "rew": -4.4027, "rew_std": 0.2059, "Agent": "TD3"}, {"env_step": 215000, "rew": -4.5156, "rew_std": 0.2303, "Agent": "TD3"}, {"env_step": 220000, "rew": -3.885, "rew_std": 0.3386, "Agent": "TD3"}, {"env_step": 225000, "rew": -4.3227, "rew_std": 0.2204, "Agent": "TD3"}, {"env_step": 230000, "rew": -4.3129, "rew_std": 0.2052, "Agent": "TD3"}, {"env_step": 235000, "rew": -3.4446, "rew_std": 0.2954, "Agent": "TD3"}, {"env_step": 240000, "rew": -4.47, "rew_std": 0.2525, "Agent": "TD3"}, {"env_step": 245000, "rew": -4.2606, "rew_std": 0.2477, "Agent": "TD3"}, {"env_step": 250000, "rew": -4.086, "rew_std": 0.4186, "Agent": "TD3"}, {"env_step": 255000, "rew": -3.5408, "rew_std": 0.273, "Agent": "TD3"}, {"env_step": 260000, "rew": -3.9576, "rew_std": 0.1268, "Agent": "TD3"}, {"env_step": 265000, "rew": -3.826, "rew_std": 0.2686, "Agent": "TD3"}, {"env_step": 270000, "rew": -3.9491, "rew_std": 0.2103, "Agent": "TD3"}, {"env_step": 275000, "rew": -3.9134, "rew_std": 0.2388, "Agent": "TD3"}, {"env_step": 280000, "rew": -3.9591, "rew_std": 0.2735, "Agent": "TD3"}, {"env_step": 285000, "rew": -4.3291, "rew_std": 0.1285, "Agent": "TD3"}, {"env_step": 290000, "rew": -3.6573, "rew_std": 0.1932, "Agent": "TD3"}, {"env_step": 295000, "rew": -3.7134, "rew_std": 0.135, "Agent": "TD3"}, {"env_step": 300000, "rew": -3.78, "rew_std": 0.3737, "Agent": "TD3"}, {"env_step": 305000, "rew": -3.8248, "rew_std": 0.1736, "Agent": "TD3"}, {"env_step": 310000, "rew": -4.18, "rew_std": 0.1973, "Agent": "TD3"}, {"env_step": 315000, "rew": -4.2059, "rew_std": 0.1313, "Agent": "TD3"}, {"env_step": 320000, "rew": -3.5267, "rew_std": 0.3696, "Agent": "TD3"}, {"env_step": 325000, "rew": -4.3458, "rew_std": 0.3499, "Agent": "TD3"}, {"env_step": 330000, "rew": -4.1234, "rew_std": 0.3483, "Agent": "TD3"}, {"env_step": 335000, "rew": -3.6075, "rew_std": 0.3162, "Agent": "TD3"}, {"env_step": 340000, "rew": -3.5364, "rew_std": 0.1874, "Agent": "TD3"}, {"env_step": 345000, "rew": -3.8618, "rew_std": 0.2639, "Agent": "TD3"}, {"env_step": 350000, "rew": -3.9008, "rew_std": 0.2792, "Agent": "TD3"}, {"env_step": 355000, "rew": -3.1281, "rew_std": 0.271, "Agent": "TD3"}, {"env_step": 360000, "rew": -3.6761, "rew_std": 0.2884, "Agent": "TD3"}, {"env_step": 365000, "rew": -4.5713, "rew_std": 0.1102, "Agent": "TD3"}, {"env_step": 370000, "rew": -3.6901, "rew_std": 0.3035, "Agent": "TD3"}, {"env_step": 375000, "rew": -4.0951, "rew_std": 0.2359, "Agent": "TD3"}, {"env_step": 380000, "rew": -4.1626, "rew_std": 0.1743, "Agent": "TD3"}, {"env_step": 385000, "rew": -3.9314, "rew_std": 0.3946, "Agent": "TD3"}, {"env_step": 390000, "rew": -4.7471, "rew_std": 0.2057, "Agent": "TD3"}, {"env_step": 395000, "rew": -3.9016, "rew_std": 0.4001, "Agent": "TD3"}, {"env_step": 400000, "rew": -3.7927, "rew_std": 0.1933, "Agent": "TD3"}, {"env_step": 405000, "rew": -4.1709, "rew_std": 0.2449, "Agent": "TD3"}, {"env_step": 410000, "rew": -4.0937, "rew_std": 0.2439, "Agent": "TD3"}, {"env_step": 415000, "rew": -3.8412, "rew_std": 0.187, "Agent": "TD3"}, {"env_step": 420000, "rew": -4.0207, "rew_std": 0.2532, "Agent": "TD3"}, {"env_step": 425000, "rew": -4.2583, "rew_std": 0.1668, "Agent": "TD3"}, {"env_step": 430000, "rew": -3.9042, "rew_std": 0.1975, "Agent": "TD3"}, {"env_step": 435000, "rew": -4.2115, "rew_std": 0.1991, "Agent": "TD3"}, {"env_step": 440000, "rew": -4.1096, "rew_std": 0.4061, "Agent": "TD3"}, {"env_step": 445000, "rew": -3.7729, "rew_std": 0.2087, "Agent": "TD3"}, {"env_step": 450000, "rew": -3.8751, "rew_std": 0.3315, "Agent": "TD3"}, {"env_step": 455000, "rew": -3.4159, "rew_std": 0.2461, "Agent": "TD3"}, {"env_step": 460000, "rew": -3.8447, "rew_std": 0.2259, "Agent": "TD3"}, {"env_step": 465000, "rew": -4.0669, "rew_std": 0.3227, "Agent": "TD3"}, {"env_step": 470000, "rew": -3.691, "rew_std": 0.2494, "Agent": "TD3"}, {"env_step": 475000, "rew": -3.3224, "rew_std": 0.3852, "Agent": "TD3"}, {"env_step": 480000, "rew": -4.0412, "rew_std": 0.2384, "Agent": "TD3"}, {"env_step": 485000, "rew": -4.0592, "rew_std": 0.3395, "Agent": "TD3"}, {"env_step": 490000, "rew": -3.2932, "rew_std": 0.2861, "Agent": "TD3"}, {"env_step": 495000, "rew": -3.9973, "rew_std": 0.1509, "Agent": "TD3"}, {"env_step": 500000, "rew": -3.8178, "rew_std": 0.2252, "Agent": "TD3"}, {"env_step": 505000, "rew": -4.2918, "rew_std": 0.3856, "Agent": "TD3"}, {"env_step": 510000, "rew": -3.8209, "rew_std": 0.2873, "Agent": "TD3"}, {"env_step": 515000, "rew": -3.9336, "rew_std": 0.2041, "Agent": "TD3"}, {"env_step": 520000, "rew": -4.0006, "rew_std": 0.215, "Agent": "TD3"}, {"env_step": 525000, "rew": -3.8263, "rew_std": 0.2583, "Agent": "TD3"}, {"env_step": 530000, "rew": -3.4156, "rew_std": 0.1398, "Agent": "TD3"}, {"env_step": 535000, "rew": -4.4681, "rew_std": 0.2342, "Agent": "TD3"}, {"env_step": 540000, "rew": -3.637, "rew_std": 0.1929, "Agent": "TD3"}, {"env_step": 545000, "rew": -3.5376, "rew_std": 0.2974, "Agent": "TD3"}, {"env_step": 550000, "rew": -4.2729, "rew_std": 0.4386, "Agent": "TD3"}, {"env_step": 555000, "rew": -3.9786, "rew_std": 0.2423, "Agent": "TD3"}, {"env_step": 560000, "rew": -3.4312, "rew_std": 0.1127, "Agent": "TD3"}, {"env_step": 565000, "rew": -4.3476, "rew_std": 0.3141, "Agent": "TD3"}, {"env_step": 570000, "rew": -3.4787, "rew_std": 0.1459, "Agent": "TD3"}, {"env_step": 575000, "rew": -4.0289, "rew_std": 0.2522, "Agent": "TD3"}, {"env_step": 580000, "rew": -3.3567, "rew_std": 0.2871, "Agent": "TD3"}, {"env_step": 585000, "rew": -3.2353, "rew_std": 0.2117, "Agent": "TD3"}, {"env_step": 590000, "rew": -3.362, "rew_std": 0.1691, "Agent": "TD3"}, {"env_step": 595000, "rew": -3.3161, "rew_std": 0.3261, "Agent": "TD3"}, {"env_step": 600000, "rew": -3.989, "rew_std": 0.2499, "Agent": "TD3"}, {"env_step": 605000, "rew": -3.9252, "rew_std": 0.1923, "Agent": "TD3"}, {"env_step": 610000, "rew": -4.7941, "rew_std": 0.3138, "Agent": "TD3"}, {"env_step": 615000, "rew": -4.0498, "rew_std": 0.2469, "Agent": "TD3"}, {"env_step": 620000, "rew": -4.5315, "rew_std": 0.0809, "Agent": "TD3"}, {"env_step": 625000, "rew": -3.6891, "rew_std": 0.4345, "Agent": "TD3"}, {"env_step": 630000, "rew": -3.6188, "rew_std": 0.4994, "Agent": "TD3"}, {"env_step": 635000, "rew": -3.5216, "rew_std": 0.1895, "Agent": "TD3"}, {"env_step": 640000, "rew": -3.6394, "rew_std": 0.2868, "Agent": "TD3"}, {"env_step": 645000, "rew": -3.8522, "rew_std": 0.2107, "Agent": "TD3"}, {"env_step": 650000, "rew": -3.1598, "rew_std": 0.3737, "Agent": "TD3"}, {"env_step": 655000, "rew": -3.939, "rew_std": 0.3151, "Agent": "TD3"}, {"env_step": 660000, "rew": -4.2649, "rew_std": 0.2185, "Agent": "TD3"}, {"env_step": 665000, "rew": -4.1026, "rew_std": 0.1811, "Agent": "TD3"}, {"env_step": 670000, "rew": -3.5874, "rew_std": 0.3452, "Agent": "TD3"}, {"env_step": 675000, "rew": -3.8099, "rew_std": 0.1105, "Agent": "TD3"}, {"env_step": 680000, "rew": -3.4478, "rew_std": 0.4465, "Agent": "TD3"}, {"env_step": 685000, "rew": -3.9152, "rew_std": 0.31, "Agent": "TD3"}, {"env_step": 690000, "rew": -3.2581, "rew_std": 0.181, "Agent": "TD3"}, {"env_step": 695000, "rew": -3.6871, "rew_std": 0.2137, "Agent": "TD3"}, {"env_step": 700000, "rew": -4.3676, "rew_std": 0.3509, "Agent": "TD3"}, {"env_step": 705000, "rew": -2.7017, "rew_std": 0.1631, "Agent": "TD3"}, {"env_step": 710000, "rew": -4.0376, "rew_std": 0.2119, "Agent": "TD3"}, {"env_step": 715000, "rew": -3.9039, "rew_std": 0.2119, "Agent": "TD3"}, {"env_step": 720000, "rew": -4.2988, "rew_std": 0.2461, "Agent": "TD3"}, {"env_step": 725000, "rew": -4.3014, "rew_std": 0.2111, "Agent": "TD3"}, {"env_step": 730000, "rew": -4.0589, "rew_std": 0.1674, "Agent": "TD3"}, {"env_step": 735000, "rew": -4.3312, "rew_std": 0.3965, "Agent": "TD3"}, {"env_step": 740000, "rew": -3.3715, "rew_std": 0.1813, "Agent": "TD3"}, {"env_step": 745000, "rew": -4.4481, "rew_std": 0.304, "Agent": "TD3"}, {"env_step": 750000, "rew": -3.7905, "rew_std": 0.3223, "Agent": "TD3"}, {"env_step": 755000, "rew": -3.9719, "rew_std": 0.2819, "Agent": "TD3"}, {"env_step": 760000, "rew": -4.2341, "rew_std": 0.1909, "Agent": "TD3"}, {"env_step": 765000, "rew": -3.9411, "rew_std": 0.348, "Agent": "TD3"}, {"env_step": 770000, "rew": -3.6554, "rew_std": 0.1726, "Agent": "TD3"}, {"env_step": 775000, "rew": -3.2103, "rew_std": 0.3632, "Agent": "TD3"}, {"env_step": 780000, "rew": -3.2774, "rew_std": 0.1973, "Agent": "TD3"}, {"env_step": 785000, "rew": -3.7977, "rew_std": 0.305, "Agent": "TD3"}, {"env_step": 790000, "rew": -3.7438, "rew_std": 0.1475, "Agent": "TD3"}, {"env_step": 795000, "rew": -4.2541, "rew_std": 0.2172, "Agent": "TD3"}, {"env_step": 800000, "rew": -4.1165, "rew_std": 0.1339, "Agent": "TD3"}, {"env_step": 805000, "rew": -3.9355, "rew_std": 0.1874, "Agent": "TD3"}, {"env_step": 810000, "rew": -4.0148, "rew_std": 0.1085, "Agent": "TD3"}, {"env_step": 815000, "rew": -3.6011, "rew_std": 0.2793, "Agent": "TD3"}, {"env_step": 820000, "rew": -4.1609, "rew_std": 0.1996, "Agent": "TD3"}, {"env_step": 825000, "rew": -3.8636, "rew_std": 0.2676, "Agent": "TD3"}, {"env_step": 830000, "rew": -3.9353, "rew_std": 0.3424, "Agent": "TD3"}, {"env_step": 835000, "rew": -3.9981, "rew_std": 0.195, "Agent": "TD3"}, {"env_step": 840000, "rew": -3.462, "rew_std": 0.2094, "Agent": "TD3"}, {"env_step": 845000, "rew": -3.9043, "rew_std": 0.2111, "Agent": "TD3"}, {"env_step": 850000, "rew": -3.8145, "rew_std": 0.344, "Agent": "TD3"}, {"env_step": 855000, "rew": -3.8187, "rew_std": 0.1498, "Agent": "TD3"}, {"env_step": 860000, "rew": -3.5996, "rew_std": 0.3091, "Agent": "TD3"}, {"env_step": 865000, "rew": -3.7927, "rew_std": 0.2665, "Agent": "TD3"}, {"env_step": 870000, "rew": -4.0968, "rew_std": 0.1984, "Agent": "TD3"}, {"env_step": 875000, "rew": -3.399, "rew_std": 0.16, "Agent": "TD3"}, {"env_step": 880000, "rew": -3.9339, "rew_std": 0.1483, "Agent": "TD3"}, {"env_step": 885000, "rew": -3.7215, "rew_std": 0.1435, "Agent": "TD3"}, {"env_step": 890000, "rew": -3.0608, "rew_std": 0.3411, "Agent": "TD3"}, {"env_step": 895000, "rew": -3.4966, "rew_std": 0.3495, "Agent": "TD3"}, {"env_step": 900000, "rew": -3.0725, "rew_std": 0.2359, "Agent": "TD3"}, {"env_step": 905000, "rew": -4.5566, "rew_std": 0.2284, "Agent": "TD3"}, {"env_step": 910000, "rew": -4.0288, "rew_std": 0.381, "Agent": "TD3"}, {"env_step": 915000, "rew": -3.9548, "rew_std": 0.2133, "Agent": "TD3"}, {"env_step": 920000, "rew": -4.0829, "rew_std": 0.3712, "Agent": "TD3"}, {"env_step": 925000, "rew": -3.356, "rew_std": 0.2326, "Agent": "TD3"}, {"env_step": 930000, "rew": -3.7336, "rew_std": 0.3556, "Agent": "TD3"}, {"env_step": 935000, "rew": -4.1592, "rew_std": 0.2811, "Agent": "TD3"}, {"env_step": 940000, "rew": -4.0395, "rew_std": 0.2811, "Agent": "TD3"}, {"env_step": 945000, "rew": -3.6727, "rew_std": 0.2061, "Agent": "TD3"}, {"env_step": 950000, "rew": -3.7449, "rew_std": 0.4064, "Agent": "TD3"}, {"env_step": 955000, "rew": -3.4838, "rew_std": 0.1478, "Agent": "TD3"}, {"env_step": 960000, "rew": -3.4326, "rew_std": 0.2707, "Agent": "TD3"}, {"env_step": 965000, "rew": -3.7454, "rew_std": 0.344, "Agent": "TD3"}, {"env_step": 970000, "rew": -4.2367, "rew_std": 0.2172, "Agent": "TD3"}, {"env_step": 975000, "rew": -4.144, "rew_std": 0.1141, "Agent": "TD3"}, {"env_step": 980000, "rew": -4.1952, "rew_std": 0.3218, "Agent": "TD3"}, {"env_step": 985000, "rew": -3.7304, "rew_std": 0.3967, "Agent": "TD3"}, {"env_step": 990000, "rew": -2.9973, "rew_std": 0.218, "Agent": "TD3"}, {"env_step": 995000, "rew": -3.234, "rew_std": 0.1885, "Agent": "TD3"}, {"env_step": 1000000, "rew": -3.8884, "rew_std": 0.272, "Agent": "TD3"}, {"env_step": 1005000, "rew": -4.1338, "rew_std": 0.2143, "Agent": "TD3"}, {"env_step": 1010000, "rew": -3.0098, "rew_std": 0.1208, "Agent": "TD3"}, {"env_step": 1015000, "rew": -4.1758, "rew_std": 0.3719, "Agent": "TD3"}, {"env_step": 1020000, "rew": -4.0399, "rew_std": 0.3482, "Agent": "TD3"}, {"env_step": 1025000, "rew": -4.2385, "rew_std": 0.2338, "Agent": "TD3"}, {"env_step": 1030000, "rew": -4.365, "rew_std": 0.2632, "Agent": "TD3"}, {"env_step": 1035000, "rew": -3.8108, "rew_std": 0.1679, "Agent": "TD3"}, {"env_step": 1040000, "rew": -4.28, "rew_std": 0.2992, "Agent": "TD3"}, {"env_step": 1045000, "rew": -3.7372, "rew_std": 0.2577, "Agent": "TD3"}, {"env_step": 1050000, "rew": -3.2307, "rew_std": 0.2714, "Agent": "TD3"}, {"env_step": 1055000, "rew": -3.9885, "rew_std": 0.4439, "Agent": "TD3"}, {"env_step": 1060000, "rew": -3.7267, "rew_std": 0.2736, "Agent": "TD3"}, {"env_step": 1065000, "rew": -4.1032, "rew_std": 0.2142, "Agent": "TD3"}, {"env_step": 1070000, "rew": -3.7962, "rew_std": 0.1767, "Agent": "TD3"}, {"env_step": 1075000, "rew": -2.8543, "rew_std": 0.3636, "Agent": "TD3"}, {"env_step": 1080000, "rew": -3.6066, "rew_std": 0.0793, "Agent": "TD3"}, {"env_step": 1085000, "rew": -3.9929, "rew_std": 0.1699, "Agent": "TD3"}, {"env_step": 1090000, "rew": -3.2463, "rew_std": 0.3051, "Agent": "TD3"}, {"env_step": 1095000, "rew": -3.5581, "rew_std": 0.2546, "Agent": "TD3"}, {"env_step": 1100000, "rew": -4.5702, "rew_std": 0.3293, "Agent": "TD3"}, {"env_step": 1105000, "rew": -3.4873, "rew_std": 0.1964, "Agent": "TD3"}, {"env_step": 1110000, "rew": -4.4129, "rew_std": 0.2345, "Agent": "TD3"}, {"env_step": 1115000, "rew": -4.0473, "rew_std": 0.2341, "Agent": "TD3"}, {"env_step": 1120000, "rew": -4.1743, "rew_std": 0.4852, "Agent": "TD3"}, {"env_step": 1125000, "rew": -3.6916, "rew_std": 0.2553, "Agent": "TD3"}, {"env_step": 1130000, "rew": -4.2822, "rew_std": 0.3409, "Agent": "TD3"}, {"env_step": 1135000, "rew": -3.4592, "rew_std": 0.1341, "Agent": "TD3"}, {"env_step": 1140000, "rew": -3.7917, "rew_std": 0.2305, "Agent": "TD3"}, {"env_step": 1145000, "rew": -3.6923, "rew_std": 0.1766, "Agent": "TD3"}, {"env_step": 1150000, "rew": -3.6304, "rew_std": 0.2707, "Agent": "TD3"}, {"env_step": 1155000, "rew": -4.0343, "rew_std": 0.179, "Agent": "TD3"}, {"env_step": 1160000, "rew": -3.2482, "rew_std": 0.2482, "Agent": "TD3"}, {"env_step": 1165000, "rew": -4.4312, "rew_std": 0.4369, "Agent": "TD3"}, {"env_step": 1170000, "rew": -4.6023, "rew_std": 0.5582, "Agent": "TD3"}, {"env_step": 1175000, "rew": -2.9858, "rew_std": 0.2303, "Agent": "TD3"}, {"env_step": 1180000, "rew": -3.4268, "rew_std": 0.1277, "Agent": "TD3"}, {"env_step": 1185000, "rew": -3.8793, "rew_std": 0.3748, "Agent": "TD3"}, {"env_step": 1190000, "rew": -4.358, "rew_std": 0.2256, "Agent": "TD3"}, {"env_step": 1195000, "rew": -3.9155, "rew_std": 0.1785, "Agent": "TD3"}, {"env_step": 1200000, "rew": -3.9947, "rew_std": 0.4121, "Agent": "TD3"}, {"env_step": 1205000, "rew": -3.6627, "rew_std": 0.1623, "Agent": "TD3"}, {"env_step": 1210000, "rew": -3.765, "rew_std": 0.4052, "Agent": "TD3"}, {"env_step": 1215000, "rew": -3.5107, "rew_std": 0.3295, "Agent": "TD3"}, {"env_step": 1220000, "rew": -4.0312, "rew_std": 0.2906, "Agent": "TD3"}, {"env_step": 1225000, "rew": -3.3748, "rew_std": 0.1568, "Agent": "TD3"}, {"env_step": 1230000, "rew": -3.8248, "rew_std": 0.1543, "Agent": "TD3"}, {"env_step": 1235000, "rew": -4.1974, "rew_std": 0.1718, "Agent": "TD3"}, {"env_step": 1240000, "rew": -3.4136, "rew_std": 0.2672, "Agent": "TD3"}, {"env_step": 1245000, "rew": -4.0437, "rew_std": 0.1481, "Agent": "TD3"}, {"env_step": 1250000, "rew": -4.1661, "rew_std": 0.2663, "Agent": "TD3"}, {"env_step": 0, "rew": -46.1947, "rew_std": 25.5804, "Agent": "TRPO"}, {"env_step": 30720, "rew": -13.9012, "rew_std": 11.913, "Agent": "TRPO"}, {"env_step": 61440, "rew": 18.0615, "rew_std": 9.0166, "Agent": "TRPO"}, {"env_step": 92160, "rew": 51.9175, "rew_std": 16.4955, "Agent": "TRPO"}, {"env_step": 122880, "rew": 108.4958, "rew_std": 33.2241, "Agent": "TRPO"}, {"env_step": 153600, "rew": 183.9944, "rew_std": 35.8262, "Agent": "TRPO"}, {"env_step": 184320, "rew": 257.4861, "rew_std": 60.051, "Agent": "TRPO"}, {"env_step": 215040, "rew": 430.4068, "rew_std": 124.2541, "Agent": "TRPO"}, {"env_step": 245760, "rew": 564.2017, "rew_std": 146.0606, "Agent": "TRPO"}, {"env_step": 276480, "rew": 682.5168, "rew_std": 122.3187, "Agent": "TRPO"}, {"env_step": 307200, "rew": 832.5033, "rew_std": 159.2968, "Agent": "TRPO"}, {"env_step": 337920, "rew": 1014.1049, "rew_std": 221.8108, "Agent": "TRPO"}, {"env_step": 368640, "rew": 1128.5618, "rew_std": 167.9213, "Agent": "TRPO"}, {"env_step": 399360, "rew": 1146.4494, "rew_std": 215.1602, "Agent": "TRPO"}, {"env_step": 430080, "rew": 1233.6374, "rew_std": 286.8604, "Agent": "TRPO"}, {"env_step": 460800, "rew": 1261.4876, "rew_std": 301.0892, "Agent": "TRPO"}, {"env_step": 491520, "rew": 1472.6865, "rew_std": 215.7148, "Agent": "TRPO"}, {"env_step": 522240, "rew": 1501.5343, "rew_std": 428.1053, "Agent": "TRPO"}, {"env_step": 552960, "rew": 1631.5161, "rew_std": 558.1692, "Agent": "TRPO"}, {"env_step": 583680, "rew": 1741.6005, "rew_std": 483.7599, "Agent": "TRPO"}, {"env_step": 614400, "rew": 1809.3667, "rew_std": 444.8329, "Agent": "TRPO"}, {"env_step": 645120, "rew": 1943.2351, "rew_std": 386.0455, "Agent": "TRPO"}, {"env_step": 675840, "rew": 2119.3593, "rew_std": 463.9853, "Agent": "TRPO"}, {"env_step": 706560, "rew": 2160.9349, "rew_std": 299.8534, "Agent": "TRPO"}, {"env_step": 737280, "rew": 2205.6584, "rew_std": 495.8226, "Agent": "TRPO"}, {"env_step": 768000, "rew": 2148.4616, "rew_std": 617.4263, "Agent": "TRPO"}, {"env_step": 798720, "rew": 2294.2347, "rew_std": 619.654, "Agent": "TRPO"}, {"env_step": 829440, "rew": 2334.3877, "rew_std": 569.8366, "Agent": "TRPO"}, {"env_step": 860160, "rew": 2373.0195, "rew_std": 635.4872, "Agent": "TRPO"}, {"env_step": 890880, "rew": 2572.1874, "rew_std": 771.2781, "Agent": "TRPO"}, {"env_step": 921600, "rew": 2568.2581, "rew_std": 539.7726, "Agent": "TRPO"}, {"env_step": 952320, "rew": 2639.2867, "rew_std": 572.4301, "Agent": "TRPO"}, {"env_step": 983040, "rew": 2866.6508, "rew_std": 707.8937, "Agent": "TRPO"}, {"env_step": 1013760, "rew": 2731.0697, "rew_std": 675.644, "Agent": "TRPO"}, {"env_step": 1044480, "rew": 2702.1965, "rew_std": 742.4807, "Agent": "TRPO"}, {"env_step": 1075200, "rew": 3001.7621, "rew_std": 688.8314, "Agent": "TRPO"}, {"env_step": 1105920, "rew": 2877.8716, "rew_std": 922.2411, "Agent": "TRPO"}, {"env_step": 1136640, "rew": 3104.9072, "rew_std": 800.8227, "Agent": "TRPO"}, {"env_step": 1167360, "rew": 2976.5528, "rew_std": 661.5153, "Agent": "TRPO"}, {"env_step": 1198080, "rew": 3196.2228, "rew_std": 707.9679, "Agent": "TRPO"}, {"env_step": 1228800, "rew": 3237.2005, "rew_std": 505.6685, "Agent": "TRPO"}, {"env_step": 1259520, "rew": 3098.3083, "rew_std": 578.4346, "Agent": "TRPO"}, {"env_step": 1290240, "rew": 3356.962, "rew_std": 786.3838, "Agent": "TRPO"}, {"env_step": 1320960, "rew": 3241.2647, "rew_std": 600.3495, "Agent": "TRPO"}, {"env_step": 1351680, "rew": 3361.1722, "rew_std": 547.5151, "Agent": "TRPO"}, {"env_step": 1382400, "rew": 3495.3879, "rew_std": 662.5387, "Agent": "TRPO"}, {"env_step": 1413120, "rew": 3388.8443, "rew_std": 537.1016, "Agent": "TRPO"}, {"env_step": 1443840, "rew": 3493.6456, "rew_std": 509.2391, "Agent": "TRPO"}, {"env_step": 1474560, "rew": 3256.3625, "rew_std": 412.1903, "Agent": "TRPO"}, {"env_step": 1505280, "rew": 3695.3862, "rew_std": 644.5284, "Agent": "TRPO"}, {"env_step": 1536000, "rew": 3685.0253, "rew_std": 537.4812, "Agent": "TRPO"}, {"env_step": 1566720, "rew": 3565.3736, "rew_std": 647.2313, "Agent": "TRPO"}, {"env_step": 1597440, "rew": 3790.5153, "rew_std": 702.1943, "Agent": "TRPO"}, {"env_step": 1628160, "rew": 3598.3267, "rew_std": 626.2541, "Agent": "TRPO"}, {"env_step": 1658880, "rew": 3970.7287, "rew_std": 647.9836, "Agent": "TRPO"}, {"env_step": 1689600, "rew": 3658.6492, "rew_std": 645.0697, "Agent": "TRPO"}, {"env_step": 1720320, "rew": 3848.0345, "rew_std": 701.1273, "Agent": "TRPO"}, {"env_step": 1751040, "rew": 4061.4513, "rew_std": 690.0289, "Agent": "TRPO"}, {"env_step": 1781760, "rew": 3807.921, "rew_std": 706.5999, "Agent": "TRPO"}, {"env_step": 1812480, "rew": 3802.0342, "rew_std": 663.7139, "Agent": "TRPO"}, {"env_step": 1843200, "rew": 3873.6662, "rew_std": 566.3833, "Agent": "TRPO"}, {"env_step": 1873920, "rew": 4038.6082, "rew_std": 518.5079, "Agent": "TRPO"}, {"env_step": 1904640, "rew": 4167.3221, "rew_std": 440.9658, "Agent": "TRPO"}, {"env_step": 1935360, "rew": 3922.2855, "rew_std": 733.8238, "Agent": "TRPO"}, {"env_step": 1966080, "rew": 4174.7307, "rew_std": 495.1849, "Agent": "TRPO"}, {"env_step": 1996800, "rew": 4155.8384, "rew_std": 477.6376, "Agent": "TRPO"}, {"env_step": 2027520, "rew": 4259.3055, "rew_std": 577.3636, "Agent": "TRPO"}, {"env_step": 2058240, "rew": 4256.1933, "rew_std": 594.2826, "Agent": "TRPO"}, {"env_step": 2088960, "rew": 4355.0669, "rew_std": 605.8978, "Agent": "TRPO"}, {"env_step": 2119680, "rew": 4297.2252, "rew_std": 474.421, "Agent": "TRPO"}, {"env_step": 2150400, "rew": 4287.4698, "rew_std": 363.7332, "Agent": "TRPO"}, {"env_step": 2181120, "rew": 4398.2137, "rew_std": 354.3603, "Agent": "TRPO"}, {"env_step": 2211840, "rew": 4413.5796, "rew_std": 460.9681, "Agent": "TRPO"}, {"env_step": 2242560, "rew": 4329.001, "rew_std": 530.2755, "Agent": "TRPO"}, {"env_step": 2273280, "rew": 4252.3464, "rew_std": 597.5882, "Agent": "TRPO"}, {"env_step": 2304000, "rew": 4462.5657, "rew_std": 495.3802, "Agent": "TRPO"}, {"env_step": 2334720, "rew": 4374.2838, "rew_std": 457.0956, "Agent": "TRPO"}, {"env_step": 2365440, "rew": 4529.4549, "rew_std": 461.1733, "Agent": "TRPO"}, {"env_step": 2396160, "rew": 4291.4468, "rew_std": 411.2142, "Agent": "TRPO"}, {"env_step": 2426880, "rew": 4430.1459, "rew_std": 601.8812, "Agent": "TRPO"}, {"env_step": 2457600, "rew": 4508.078, "rew_std": 465.0507, "Agent": "TRPO"}, {"env_step": 2488320, "rew": 4151.3796, "rew_std": 414.656, "Agent": "TRPO"}, {"env_step": 2519040, "rew": 4705.7212, "rew_std": 477.1206, "Agent": "TRPO"}, {"env_step": 2549760, "rew": 4641.3111, "rew_std": 622.6369, "Agent": "TRPO"}, {"env_step": 2580480, "rew": 4484.1452, "rew_std": 442.0811, "Agent": "TRPO"}, {"env_step": 2611200, "rew": 4658.7294, "rew_std": 432.1496, "Agent": "TRPO"}, {"env_step": 2641920, "rew": 4625.8785, "rew_std": 357.4208, "Agent": "TRPO"}, {"env_step": 2672640, "rew": 4284.9794, "rew_std": 599.7262, "Agent": "TRPO"}, {"env_step": 2703360, "rew": 4419.1972, "rew_std": 447.5325, "Agent": "TRPO"}, {"env_step": 2734080, "rew": 4746.1067, "rew_std": 580.4515, "Agent": "TRPO"}, {"env_step": 2764800, "rew": 4756.6644, "rew_std": 502.4658, "Agent": "TRPO"}, {"env_step": 2795520, "rew": 4607.7287, "rew_std": 427.1984, "Agent": "TRPO"}, {"env_step": 2826240, "rew": 4404.4363, "rew_std": 555.3297, "Agent": "TRPO"}, {"env_step": 2856960, "rew": 4816.8623, "rew_std": 528.6794, "Agent": "TRPO"}, {"env_step": 2887680, "rew": 4753.3578, "rew_std": 715.7057, "Agent": "TRPO"}, {"env_step": 2918400, "rew": 4604.763, "rew_std": 447.4458, "Agent": "TRPO"}, {"env_step": 2949120, "rew": 4800.2693, "rew_std": 602.9667, "Agent": "TRPO"}, {"env_step": 2979840, "rew": 4772.9396, "rew_std": 368.1691, "Agent": "TRPO"}, {"env_step": 3010560, "rew": 4897.0257, "rew_std": 449.1329, "Agent": "TRPO"}, {"env_step": 3041280, "rew": 4841.7814, "rew_std": 534.8951, "Agent": "TRPO"}, {"env_step": 3072000, "rew": 4604.9043, "rew_std": 580.976, "Agent": "TRPO"}, {"env_step": 0, "rew": -46.1947, "rew_std": 25.5804, "Agent": "NPG"}, {"env_step": 30720, "rew": -29.0009, "rew_std": 15.8916, "Agent": "NPG"}, {"env_step": 61440, "rew": -10.4279, "rew_std": 11.0242, "Agent": "NPG"}, {"env_step": 92160, "rew": 11.1187, "rew_std": 9.0737, "Agent": "NPG"}, {"env_step": 122880, "rew": 27.5852, "rew_std": 16.4638, "Agent": "NPG"}, {"env_step": 153600, "rew": 56.7285, "rew_std": 23.8642, "Agent": "NPG"}, {"env_step": 184320, "rew": 81.2319, "rew_std": 29.1525, "Agent": "NPG"}, {"env_step": 215040, "rew": 159.4704, "rew_std": 47.3793, "Agent": "NPG"}, {"env_step": 245760, "rew": 174.9781, "rew_std": 57.693, "Agent": "NPG"}, {"env_step": 276480, "rew": 178.6358, "rew_std": 32.807, "Agent": "NPG"}, {"env_step": 307200, "rew": 218.3212, "rew_std": 41.035, "Agent": "NPG"}, {"env_step": 337920, "rew": 368.663, "rew_std": 80.0623, "Agent": "NPG"}, {"env_step": 368640, "rew": 432.2163, "rew_std": 63.1225, "Agent": "NPG"}, {"env_step": 399360, "rew": 568.0807, "rew_std": 125.517, "Agent": "NPG"}, {"env_step": 430080, "rew": 631.1259, "rew_std": 114.6371, "Agent": "NPG"}, {"env_step": 460800, "rew": 753.5929, "rew_std": 118.5185, "Agent": "NPG"}, {"env_step": 491520, "rew": 864.7371, "rew_std": 183.4071, "Agent": "NPG"}, {"env_step": 522240, "rew": 940.2978, "rew_std": 158.8018, "Agent": "NPG"}, {"env_step": 552960, "rew": 1021.7661, "rew_std": 159.2406, "Agent": "NPG"}, {"env_step": 583680, "rew": 1131.714, "rew_std": 150.4756, "Agent": "NPG"}, {"env_step": 614400, "rew": 1201.5181, "rew_std": 164.258, "Agent": "NPG"}, {"env_step": 645120, "rew": 1343.1191, "rew_std": 184.6686, "Agent": "NPG"}, {"env_step": 675840, "rew": 1318.401, "rew_std": 229.7468, "Agent": "NPG"}, {"env_step": 706560, "rew": 1459.1159, "rew_std": 222.6298, "Agent": "NPG"}, {"env_step": 737280, "rew": 1542.0295, "rew_std": 203.9479, "Agent": "NPG"}, {"env_step": 768000, "rew": 1634.9176, "rew_std": 218.2131, "Agent": "NPG"}, {"env_step": 798720, "rew": 1692.0983, "rew_std": 343.9785, "Agent": "NPG"}, {"env_step": 829440, "rew": 1906.1507, "rew_std": 457.5102, "Agent": "NPG"}, {"env_step": 860160, "rew": 1775.1748, "rew_std": 342.0144, "Agent": "NPG"}, {"env_step": 890880, "rew": 2102.0494, "rew_std": 355.8503, "Agent": "NPG"}, {"env_step": 921600, "rew": 2046.7794, "rew_std": 340.8884, "Agent": "NPG"}, {"env_step": 952320, "rew": 2164.3512, "rew_std": 405.7482, "Agent": "NPG"}, {"env_step": 983040, "rew": 2348.312, "rew_std": 509.9401, "Agent": "NPG"}, {"env_step": 1013760, "rew": 2365.8958, "rew_std": 523.6603, "Agent": "NPG"}, {"env_step": 1044480, "rew": 2324.9164, "rew_std": 528.4526, "Agent": "NPG"}, {"env_step": 1075200, "rew": 2493.722, "rew_std": 583.8895, "Agent": "NPG"}, {"env_step": 1105920, "rew": 2638.7282, "rew_std": 650.3356, "Agent": "NPG"}, {"env_step": 1136640, "rew": 2588.3598, "rew_std": 506.4239, "Agent": "NPG"}, {"env_step": 1167360, "rew": 2657.8218, "rew_std": 542.1361, "Agent": "NPG"}, {"env_step": 1198080, "rew": 2772.0887, "rew_std": 399.8722, "Agent": "NPG"}, {"env_step": 1228800, "rew": 2983.8532, "rew_std": 405.1427, "Agent": "NPG"}, {"env_step": 1259520, "rew": 2969.912, "rew_std": 543.0224, "Agent": "NPG"}, {"env_step": 1290240, "rew": 2898.5856, "rew_std": 687.1927, "Agent": "NPG"}, {"env_step": 1320960, "rew": 2979.687, "rew_std": 685.3181, "Agent": "NPG"}, {"env_step": 1351680, "rew": 2937.7865, "rew_std": 654.2811, "Agent": "NPG"}, {"env_step": 1382400, "rew": 3180.3262, "rew_std": 502.1675, "Agent": "NPG"}, {"env_step": 1413120, "rew": 3181.4746, "rew_std": 526.9918, "Agent": "NPG"}, {"env_step": 1443840, "rew": 3186.725, "rew_std": 483.0104, "Agent": "NPG"}, {"env_step": 1474560, "rew": 3347.1624, "rew_std": 474.3212, "Agent": "NPG"}, {"env_step": 1505280, "rew": 3264.1247, "rew_std": 542.1326, "Agent": "NPG"}, {"env_step": 1536000, "rew": 3267.3737, "rew_std": 536.477, "Agent": "NPG"}, {"env_step": 1566720, "rew": 3677.709, "rew_std": 486.0857, "Agent": "NPG"}, {"env_step": 1597440, "rew": 3379.1083, "rew_std": 683.1763, "Agent": "NPG"}, {"env_step": 1628160, "rew": 3533.6653, "rew_std": 437.9623, "Agent": "NPG"}, {"env_step": 1658880, "rew": 3534.0292, "rew_std": 787.6867, "Agent": "NPG"}, {"env_step": 1689600, "rew": 3399.7589, "rew_std": 685.1419, "Agent": "NPG"}, {"env_step": 1720320, "rew": 3587.7024, "rew_std": 471.134, "Agent": "NPG"}, {"env_step": 1751040, "rew": 3560.6233, "rew_std": 616.8093, "Agent": "NPG"}, {"env_step": 1781760, "rew": 3635.7303, "rew_std": 608.4666, "Agent": "NPG"}, {"env_step": 1812480, "rew": 3713.5899, "rew_std": 802.8577, "Agent": "NPG"}, {"env_step": 1843200, "rew": 3590.908, "rew_std": 557.7779, "Agent": "NPG"}, {"env_step": 1873920, "rew": 3903.7812, "rew_std": 655.7805, "Agent": "NPG"}, {"env_step": 1904640, "rew": 3770.5075, "rew_std": 645.9229, "Agent": "NPG"}, {"env_step": 1935360, "rew": 4110.3599, "rew_std": 657.3085, "Agent": "NPG"}, {"env_step": 1966080, "rew": 3851.8914, "rew_std": 745.2079, "Agent": "NPG"}, {"env_step": 1996800, "rew": 3998.0014, "rew_std": 650.7886, "Agent": "NPG"}, {"env_step": 2027520, "rew": 3739.3657, "rew_std": 715.6727, "Agent": "NPG"}, {"env_step": 2058240, "rew": 4230.5764, "rew_std": 578.5901, "Agent": "NPG"}, {"env_step": 2088960, "rew": 4140.935, "rew_std": 624.3245, "Agent": "NPG"}, {"env_step": 2119680, "rew": 4228.9808, "rew_std": 573.9637, "Agent": "NPG"}, {"env_step": 2150400, "rew": 4142.6649, "rew_std": 419.5781, "Agent": "NPG"}, {"env_step": 2181120, "rew": 4089.8147, "rew_std": 302.9776, "Agent": "NPG"}, {"env_step": 2211840, "rew": 4276.8854, "rew_std": 674.4424, "Agent": "NPG"}, {"env_step": 2242560, "rew": 4312.1654, "rew_std": 567.5481, "Agent": "NPG"}, {"env_step": 2273280, "rew": 4224.3566, "rew_std": 445.0954, "Agent": "NPG"}, {"env_step": 2304000, "rew": 3878.7632, "rew_std": 866.3551, "Agent": "NPG"}, {"env_step": 2334720, "rew": 4001.3801, "rew_std": 478.1705, "Agent": "NPG"}, {"env_step": 2365440, "rew": 4284.5594, "rew_std": 448.8121, "Agent": "NPG"}, {"env_step": 2396160, "rew": 4406.6923, "rew_std": 516.8683, "Agent": "NPG"}, {"env_step": 2426880, "rew": 4145.9032, "rew_std": 519.4268, "Agent": "NPG"}, {"env_step": 2457600, "rew": 4241.7632, "rew_std": 685.6791, "Agent": "NPG"}, {"env_step": 2488320, "rew": 4154.1219, "rew_std": 437.1567, "Agent": "NPG"}, {"env_step": 2519040, "rew": 4495.7374, "rew_std": 449.1806, "Agent": "NPG"}, {"env_step": 2549760, "rew": 4289.7487, "rew_std": 526.7084, "Agent": "NPG"}, {"env_step": 2580480, "rew": 4246.0211, "rew_std": 530.6265, "Agent": "NPG"}, {"env_step": 2611200, "rew": 4463.8985, "rew_std": 572.8797, "Agent": "NPG"}, {"env_step": 2641920, "rew": 4298.6282, "rew_std": 665.9416, "Agent": "NPG"}, {"env_step": 2672640, "rew": 4485.5394, "rew_std": 359.9158, "Agent": "NPG"}, {"env_step": 2703360, "rew": 4820.597, "rew_std": 318.5627, "Agent": "NPG"}, {"env_step": 2734080, "rew": 4540.1727, "rew_std": 412.9545, "Agent": "NPG"}, {"env_step": 2764800, "rew": 4500.1939, "rew_std": 596.1343, "Agent": "NPG"}, {"env_step": 2795520, "rew": 4664.0692, "rew_std": 426.6286, "Agent": "NPG"}, {"env_step": 2826240, "rew": 4516.467, "rew_std": 435.656, "Agent": "NPG"}, {"env_step": 2856960, "rew": 4545.139, "rew_std": 496.411, "Agent": "NPG"}, {"env_step": 2887680, "rew": 4444.0518, "rew_std": 596.6213, "Agent": "NPG"}, {"env_step": 2918400, "rew": 4658.8307, "rew_std": 652.0561, "Agent": "NPG"}, {"env_step": 2949120, "rew": 4509.4439, "rew_std": 466.3483, "Agent": "NPG"}, {"env_step": 2979840, "rew": 4589.8375, "rew_std": 414.0363, "Agent": "NPG"}, {"env_step": 3010560, "rew": 4693.0902, "rew_std": 490.4674, "Agent": "NPG"}, {"env_step": 3041280, "rew": 4678.8152, "rew_std": 313.0054, "Agent": "NPG"}, {"env_step": 3072000, "rew": 4736.6273, "rew_std": 376.7205, "Agent": "NPG"}, {"env_step": 0, "rew": -39.7281, "rew_std": 1.6637, "Agent": "PPO"}, {"env_step": 30720, "rew": -25.1977, "rew_std": 1.9758, "Agent": "PPO"}, {"env_step": 61440, "rew": -11.3756, "rew_std": 0.5121, "Agent": "PPO"}, {"env_step": 92160, "rew": -9.4285, "rew_std": 0.4521, "Agent": "PPO"}, {"env_step": 122880, "rew": -5.9949, "rew_std": 1.0133, "Agent": "PPO"}, {"env_step": 153600, "rew": -6.0315, "rew_std": 0.4946, "Agent": "PPO"}, {"env_step": 184320, "rew": -6.1742, "rew_std": 0.5615, "Agent": "PPO"}, {"env_step": 215040, "rew": -6.3098, "rew_std": 0.9039, "Agent": "PPO"}, {"env_step": 245760, "rew": -5.5179, "rew_std": 0.5696, "Agent": "PPO"}, {"env_step": 276480, "rew": -6.2265, "rew_std": 1.0758, "Agent": "PPO"}, {"env_step": 307200, "rew": -5.0218, "rew_std": 0.5475, "Agent": "PPO"}, {"env_step": 337920, "rew": -5.4688, "rew_std": 0.5576, "Agent": "PPO"}, {"env_step": 368640, "rew": -5.8007, "rew_std": 0.5325, "Agent": "PPO"}, {"env_step": 399360, "rew": -5.4248, "rew_std": 0.4431, "Agent": "PPO"}, {"env_step": 430080, "rew": -5.3748, "rew_std": 0.5162, "Agent": "PPO"}, {"env_step": 460800, "rew": -4.9695, "rew_std": 0.3465, "Agent": "PPO"}, {"env_step": 491520, "rew": -5.0357, "rew_std": 0.4891, "Agent": "PPO"}, {"env_step": 522240, "rew": -5.5683, "rew_std": 0.8495, "Agent": "PPO"}, {"env_step": 552960, "rew": -5.3753, "rew_std": 0.4371, "Agent": "PPO"}, {"env_step": 583680, "rew": -5.884, "rew_std": 0.4708, "Agent": "PPO"}, {"env_step": 614400, "rew": -5.5962, "rew_std": 0.3539, "Agent": "PPO"}, {"env_step": 645120, "rew": -5.3307, "rew_std": 0.4992, "Agent": "PPO"}, {"env_step": 675840, "rew": -6.0082, "rew_std": 0.9148, "Agent": "PPO"}, {"env_step": 706560, "rew": -5.2822, "rew_std": 0.4585, "Agent": "PPO"}, {"env_step": 737280, "rew": -4.7864, "rew_std": 0.7392, "Agent": "PPO"}, {"env_step": 768000, "rew": -5.0164, "rew_std": 0.3031, "Agent": "PPO"}, {"env_step": 798720, "rew": -5.3032, "rew_std": 0.5742, "Agent": "PPO"}, {"env_step": 829440, "rew": -5.1033, "rew_std": 0.3818, "Agent": "PPO"}, {"env_step": 860160, "rew": -5.3432, "rew_std": 0.8027, "Agent": "PPO"}, {"env_step": 890880, "rew": -4.6837, "rew_std": 0.5437, "Agent": "PPO"}, {"env_step": 921600, "rew": -4.7674, "rew_std": 0.3104, "Agent": "PPO"}, {"env_step": 952320, "rew": -4.1239, "rew_std": 0.3437, "Agent": "PPO"}, {"env_step": 983040, "rew": -4.595, "rew_std": 0.5033, "Agent": "PPO"}, {"env_step": 1013760, "rew": -4.9547, "rew_std": 0.5945, "Agent": "PPO"}, {"env_step": 1044480, "rew": -4.4337, "rew_std": 0.5052, "Agent": "PPO"}, {"env_step": 1075200, "rew": -4.6478, "rew_std": 0.377, "Agent": "PPO"}, {"env_step": 1105920, "rew": -4.6753, "rew_std": 0.5188, "Agent": "PPO"}, {"env_step": 1136640, "rew": -4.7786, "rew_std": 0.5445, "Agent": "PPO"}, {"env_step": 1167360, "rew": -4.7328, "rew_std": 0.5834, "Agent": "PPO"}, {"env_step": 1198080, "rew": -4.5149, "rew_std": 0.3676, "Agent": "PPO"}, {"env_step": 1228800, "rew": -4.8816, "rew_std": 0.402, "Agent": "PPO"}, {"env_step": 1259520, "rew": -5.1436, "rew_std": 0.4365, "Agent": "PPO"}, {"env_step": 1290240, "rew": -4.668, "rew_std": 0.3933, "Agent": "PPO"}, {"env_step": 1320960, "rew": -5.2896, "rew_std": 0.8658, "Agent": "PPO"}, {"env_step": 1351680, "rew": -4.5049, "rew_std": 0.5145, "Agent": "PPO"}, {"env_step": 1382400, "rew": -4.1827, "rew_std": 0.3268, "Agent": "PPO"}, {"env_step": 1413120, "rew": -4.3491, "rew_std": 0.8491, "Agent": "PPO"}, {"env_step": 1443840, "rew": -3.6947, "rew_std": 0.3381, "Agent": "PPO"}, {"env_step": 1474560, "rew": -4.4419, "rew_std": 0.3714, "Agent": "PPO"}, {"env_step": 1505280, "rew": -4.3424, "rew_std": 0.4925, "Agent": "PPO"}, {"env_step": 1536000, "rew": -4.2711, "rew_std": 0.2769, "Agent": "PPO"}, {"env_step": 1566720, "rew": -4.6186, "rew_std": 0.4963, "Agent": "PPO"}, {"env_step": 1597440, "rew": -4.625, "rew_std": 0.2289, "Agent": "PPO"}, {"env_step": 1628160, "rew": -4.3231, "rew_std": 0.4513, "Agent": "PPO"}, {"env_step": 1658880, "rew": -4.1184, "rew_std": 0.4354, "Agent": "PPO"}, {"env_step": 1689600, "rew": -4.4786, "rew_std": 0.4347, "Agent": "PPO"}, {"env_step": 1720320, "rew": -3.823, "rew_std": 0.4088, "Agent": "PPO"}, {"env_step": 1751040, "rew": -4.5142, "rew_std": 0.5797, "Agent": "PPO"}, {"env_step": 1781760, "rew": -3.9392, "rew_std": 0.4262, "Agent": "PPO"}, {"env_step": 1812480, "rew": -4.455, "rew_std": 0.3795, "Agent": "PPO"}, {"env_step": 1843200, "rew": -4.309, "rew_std": 0.3757, "Agent": "PPO"}, {"env_step": 1873920, "rew": -4.4861, "rew_std": 0.4123, "Agent": "PPO"}, {"env_step": 1904640, "rew": -3.8674, "rew_std": 0.2822, "Agent": "PPO"}, {"env_step": 1935360, "rew": -3.7656, "rew_std": 0.438, "Agent": "PPO"}, {"env_step": 1966080, "rew": -4.5965, "rew_std": 0.3614, "Agent": "PPO"}, {"env_step": 1996800, "rew": -3.847, "rew_std": 0.3441, "Agent": "PPO"}, {"env_step": 2027520, "rew": -4.1442, "rew_std": 0.2726, "Agent": "PPO"}, {"env_step": 2058240, "rew": -4.81, "rew_std": 0.5441, "Agent": "PPO"}, {"env_step": 2088960, "rew": -4.489, "rew_std": 0.3588, "Agent": "PPO"}, {"env_step": 2119680, "rew": -4.6354, "rew_std": 0.3608, "Agent": "PPO"}, {"env_step": 2150400, "rew": -4.8927, "rew_std": 0.3568, "Agent": "PPO"}, {"env_step": 2181120, "rew": -4.9722, "rew_std": 0.3395, "Agent": "PPO"}, {"env_step": 2211840, "rew": -3.9921, "rew_std": 0.4937, "Agent": "PPO"}, {"env_step": 2242560, "rew": -4.8395, "rew_std": 0.5691, "Agent": "PPO"}, {"env_step": 2273280, "rew": -4.2018, "rew_std": 0.2426, "Agent": "PPO"}, {"env_step": 2304000, "rew": -4.9258, "rew_std": 0.4614, "Agent": "PPO"}, {"env_step": 2334720, "rew": -4.4081, "rew_std": 0.2367, "Agent": "PPO"}, {"env_step": 2365440, "rew": -3.9365, "rew_std": 0.5545, "Agent": "PPO"}, {"env_step": 2396160, "rew": -4.6383, "rew_std": 0.2892, "Agent": "PPO"}, {"env_step": 2426880, "rew": -3.7809, "rew_std": 0.461, "Agent": "PPO"}, {"env_step": 2457600, "rew": -4.5572, "rew_std": 0.3636, "Agent": "PPO"}, {"env_step": 2488320, "rew": -5.2924, "rew_std": 0.3211, "Agent": "PPO"}, {"env_step": 2519040, "rew": -4.2562, "rew_std": 0.4444, "Agent": "PPO"}, {"env_step": 2549760, "rew": -4.0844, "rew_std": 0.4715, "Agent": "PPO"}, {"env_step": 2580480, "rew": -4.6425, "rew_std": 0.2153, "Agent": "PPO"}, {"env_step": 2611200, "rew": -4.1332, "rew_std": 0.5231, "Agent": "PPO"}, {"env_step": 2641920, "rew": -3.811, "rew_std": 0.4255, "Agent": "PPO"}, {"env_step": 2672640, "rew": -4.4177, "rew_std": 0.3892, "Agent": "PPO"}, {"env_step": 2703360, "rew": -3.9716, "rew_std": 0.5346, "Agent": "PPO"}, {"env_step": 2734080, "rew": -4.1433, "rew_std": 0.3098, "Agent": "PPO"}, {"env_step": 2764800, "rew": -4.4922, "rew_std": 0.4792, "Agent": "PPO"}, {"env_step": 2795520, "rew": -4.3185, "rew_std": 0.3626, "Agent": "PPO"}, {"env_step": 2826240, "rew": -4.077, "rew_std": 0.3572, "Agent": "PPO"}, {"env_step": 2856960, "rew": -4.9316, "rew_std": 0.4112, "Agent": "PPO"}, {"env_step": 2887680, "rew": -4.1553, "rew_std": 0.1359, "Agent": "PPO"}, {"env_step": 2918400, "rew": -4.3514, "rew_std": 0.2179, "Agent": "PPO"}, {"env_step": 2949120, "rew": -4.1512, "rew_std": 0.226, "Agent": "PPO"}, {"env_step": 2979840, "rew": -4.4676, "rew_std": 0.2695, "Agent": "PPO"}, {"env_step": 3010560, "rew": -4.1195, "rew_std": 0.2915, "Agent": "PPO"}, {"env_step": 3041280, "rew": -4.924, "rew_std": 0.4478, "Agent": "PPO"}, {"env_step": 3072000, "rew": -4.7199, "rew_std": 0.261, "Agent": "PPO"}, {"env_step": 0, "rew": -31.7242, "rew_std": 1.1515, "Agent": "REINFORCE"}, {"env_step": 30720, "rew": -32.8475, "rew_std": 0.7602, "Agent": "REINFORCE"}, {"env_step": 61440, "rew": -31.4946, "rew_std": 0.9164, "Agent": "REINFORCE"}, {"env_step": 92160, "rew": -31.7964, "rew_std": 0.8369, "Agent": "REINFORCE"}, {"env_step": 122880, "rew": -31.0148, "rew_std": 0.9304, "Agent": "REINFORCE"}, {"env_step": 153600, "rew": -30.2386, "rew_std": 0.9128, "Agent": "REINFORCE"}, {"env_step": 184320, "rew": -30.2757, "rew_std": 0.867, "Agent": "REINFORCE"}, {"env_step": 215040, "rew": -30.1909, "rew_std": 1.1399, "Agent": "REINFORCE"}, {"env_step": 245760, "rew": -29.6005, "rew_std": 1.1925, "Agent": "REINFORCE"}, {"env_step": 276480, "rew": -29.2278, "rew_std": 0.861, "Agent": "REINFORCE"}, {"env_step": 307200, "rew": -27.9807, "rew_std": 0.6982, "Agent": "REINFORCE"}, {"env_step": 337920, "rew": -28.2742, "rew_std": 0.692, "Agent": "REINFORCE"}, {"env_step": 368640, "rew": -29.0547, "rew_std": 1.4016, "Agent": "REINFORCE"}, {"env_step": 399360, "rew": -27.0411, "rew_std": 1.2675, "Agent": "REINFORCE"}, {"env_step": 430080, "rew": -27.1825, "rew_std": 0.5854, "Agent": "REINFORCE"}, {"env_step": 460800, "rew": -27.4172, "rew_std": 1.2445, "Agent": "REINFORCE"}, {"env_step": 491520, "rew": -26.1982, "rew_std": 0.8664, "Agent": "REINFORCE"}, {"env_step": 522240, "rew": -26.4559, "rew_std": 1.0527, "Agent": "REINFORCE"}, {"env_step": 552960, "rew": -26.471, "rew_std": 0.7768, "Agent": "REINFORCE"}, {"env_step": 583680, "rew": -25.7617, "rew_std": 1.0321, "Agent": "REINFORCE"}, {"env_step": 614400, "rew": -26.0457, "rew_std": 0.7244, "Agent": "REINFORCE"}, {"env_step": 645120, "rew": -25.0898, "rew_std": 1.1499, "Agent": "REINFORCE"}, {"env_step": 675840, "rew": -25.8386, "rew_std": 0.9712, "Agent": "REINFORCE"}, {"env_step": 706560, "rew": -24.9295, "rew_std": 1.1758, "Agent": "REINFORCE"}, {"env_step": 737280, "rew": -23.7776, "rew_std": 0.8549, "Agent": "REINFORCE"}, {"env_step": 768000, "rew": -24.0209, "rew_std": 0.968, "Agent": "REINFORCE"}, {"env_step": 798720, "rew": -23.2574, "rew_std": 1.0228, "Agent": "REINFORCE"}, {"env_step": 829440, "rew": -23.4654, "rew_std": 1.1348, "Agent": "REINFORCE"}, {"env_step": 860160, "rew": -23.1139, "rew_std": 1.1578, "Agent": "REINFORCE"}, {"env_step": 890880, "rew": -21.8823, "rew_std": 1.0453, "Agent": "REINFORCE"}, {"env_step": 921600, "rew": -21.9515, "rew_std": 0.8806, "Agent": "REINFORCE"}, {"env_step": 952320, "rew": -21.6887, "rew_std": 1.4182, "Agent": "REINFORCE"}, {"env_step": 983040, "rew": -21.4815, "rew_std": 0.7857, "Agent": "REINFORCE"}, {"env_step": 1013760, "rew": -20.9373, "rew_std": 1.0882, "Agent": "REINFORCE"}, {"env_step": 1044480, "rew": -21.2495, "rew_std": 1.0276, "Agent": "REINFORCE"}, {"env_step": 1075200, "rew": -20.0153, "rew_std": 0.7888, "Agent": "REINFORCE"}, {"env_step": 1105920, "rew": -20.4975, "rew_std": 1.0074, "Agent": "REINFORCE"}, {"env_step": 1136640, "rew": -20.146, "rew_std": 1.0807, "Agent": "REINFORCE"}, {"env_step": 1167360, "rew": -19.7466, "rew_std": 0.796, "Agent": "REINFORCE"}, {"env_step": 1198080, "rew": -18.9596, "rew_std": 0.9519, "Agent": "REINFORCE"}, {"env_step": 1228800, "rew": -20.5227, "rew_std": 1.0082, "Agent": "REINFORCE"}, {"env_step": 1259520, "rew": -18.945, "rew_std": 1.1814, "Agent": "REINFORCE"}, {"env_step": 1290240, "rew": -18.4987, "rew_std": 0.6713, "Agent": "REINFORCE"}, {"env_step": 1320960, "rew": -19.6578, "rew_std": 0.9521, "Agent": "REINFORCE"}, {"env_step": 1351680, "rew": -17.8018, "rew_std": 0.8241, "Agent": "REINFORCE"}, {"env_step": 1382400, "rew": -17.8281, "rew_std": 0.8157, "Agent": "REINFORCE"}, {"env_step": 1413120, "rew": -17.0201, "rew_std": 1.0223, "Agent": "REINFORCE"}, {"env_step": 1443840, "rew": -16.9074, "rew_std": 1.4043, "Agent": "REINFORCE"}, {"env_step": 1474560, "rew": -17.6015, "rew_std": 1.4143, "Agent": "REINFORCE"}, {"env_step": 1505280, "rew": -16.7421, "rew_std": 0.9056, "Agent": "REINFORCE"}, {"env_step": 1536000, "rew": -16.1835, "rew_std": 0.9015, "Agent": "REINFORCE"}, {"env_step": 1566720, "rew": -17.2971, "rew_std": 1.0613, "Agent": "REINFORCE"}, {"env_step": 1597440, "rew": -15.9043, "rew_std": 0.9545, "Agent": "REINFORCE"}, {"env_step": 1628160, "rew": -16.0512, "rew_std": 1.1131, "Agent": "REINFORCE"}, {"env_step": 1658880, "rew": -15.4652, "rew_std": 0.9131, "Agent": "REINFORCE"}, {"env_step": 1689600, "rew": -15.246, "rew_std": 1.0308, "Agent": "REINFORCE"}, {"env_step": 1720320, "rew": -14.9502, "rew_std": 1.1819, "Agent": "REINFORCE"}, {"env_step": 1751040, "rew": -15.1105, "rew_std": 0.8577, "Agent": "REINFORCE"}, {"env_step": 1781760, "rew": -14.5965, "rew_std": 0.9296, "Agent": "REINFORCE"}, {"env_step": 1812480, "rew": -14.8261, "rew_std": 0.8252, "Agent": "REINFORCE"}, {"env_step": 1843200, "rew": -14.1788, "rew_std": 0.5964, "Agent": "REINFORCE"}, {"env_step": 1873920, "rew": -14.5974, "rew_std": 0.9389, "Agent": "REINFORCE"}, {"env_step": 1904640, "rew": -13.6165, "rew_std": 1.0934, "Agent": "REINFORCE"}, {"env_step": 1935360, "rew": -13.5211, "rew_std": 0.8089, "Agent": "REINFORCE"}, {"env_step": 1966080, "rew": -14.1801, "rew_std": 0.6711, "Agent": "REINFORCE"}, {"env_step": 1996800, "rew": -13.1272, "rew_std": 0.7476, "Agent": "REINFORCE"}, {"env_step": 2027520, "rew": -13.2716, "rew_std": 0.8896, "Agent": "REINFORCE"}, {"env_step": 2058240, "rew": -14.2973, "rew_std": 0.7262, "Agent": "REINFORCE"}, {"env_step": 2088960, "rew": -13.4797, "rew_std": 0.9319, "Agent": "REINFORCE"}, {"env_step": 2119680, "rew": -13.3317, "rew_std": 0.7338, "Agent": "REINFORCE"}, {"env_step": 2150400, "rew": -13.731, "rew_std": 0.6122, "Agent": "REINFORCE"}, {"env_step": 2181120, "rew": -13.9919, "rew_std": 0.7474, "Agent": "REINFORCE"}, {"env_step": 2211840, "rew": -12.7447, "rew_std": 0.6264, "Agent": "REINFORCE"}, {"env_step": 2242560, "rew": -13.8354, "rew_std": 0.7329, "Agent": "REINFORCE"}, {"env_step": 2273280, "rew": -12.781, "rew_std": 0.5716, "Agent": "REINFORCE"}, {"env_step": 2304000, "rew": -13.3988, "rew_std": 0.5085, "Agent": "REINFORCE"}, {"env_step": 2334720, "rew": -12.2226, "rew_std": 0.5718, "Agent": "REINFORCE"}, {"env_step": 2365440, "rew": -11.5359, "rew_std": 0.6257, "Agent": "REINFORCE"}, {"env_step": 2396160, "rew": -12.9682, "rew_std": 0.6416, "Agent": "REINFORCE"}, {"env_step": 2426880, "rew": -11.5897, "rew_std": 0.8087, "Agent": "REINFORCE"}, {"env_step": 2457600, "rew": -12.1057, "rew_std": 0.9083, "Agent": "REINFORCE"}, {"env_step": 2488320, "rew": -13.4054, "rew_std": 0.5202, "Agent": "REINFORCE"}, {"env_step": 2519040, "rew": -11.6246, "rew_std": 1.0265, "Agent": "REINFORCE"}, {"env_step": 2549760, "rew": -11.6709, "rew_std": 0.6445, "Agent": "REINFORCE"}, {"env_step": 2580480, "rew": -11.9136, "rew_std": 0.8746, "Agent": "REINFORCE"}, {"env_step": 2611200, "rew": -11.4543, "rew_std": 0.5406, "Agent": "REINFORCE"}, {"env_step": 2641920, "rew": -10.8488, "rew_std": 0.7072, "Agent": "REINFORCE"}, {"env_step": 2672640, "rew": -11.9148, "rew_std": 0.6987, "Agent": "REINFORCE"}, {"env_step": 2703360, "rew": -10.8701, "rew_std": 0.6498, "Agent": "REINFORCE"}, {"env_step": 2734080, "rew": -11.1025, "rew_std": 0.7378, "Agent": "REINFORCE"}, {"env_step": 2764800, "rew": -11.403, "rew_std": 0.7242, "Agent": "REINFORCE"}, {"env_step": 2795520, "rew": -11.4606, "rew_std": 1.0069, "Agent": "REINFORCE"}, {"env_step": 2826240, "rew": -10.8376, "rew_std": 0.3489, "Agent": "REINFORCE"}, {"env_step": 2856960, "rew": -12.1351, "rew_std": 0.6699, "Agent": "REINFORCE"}, {"env_step": 2887680, "rew": -10.9223, "rew_std": 0.5095, "Agent": "REINFORCE"}, {"env_step": 2918400, "rew": -11.001, "rew_std": 0.4022, "Agent": "REINFORCE"}, {"env_step": 2949120, "rew": -10.5209, "rew_std": 0.662, "Agent": "REINFORCE"}, {"env_step": 2979840, "rew": -11.2583, "rew_std": 0.6426, "Agent": "REINFORCE"}, {"env_step": 3010560, "rew": -10.5418, "rew_std": 0.5359, "Agent": "REINFORCE"}, {"env_step": 3041280, "rew": -11.3974, "rew_std": 0.4321, "Agent": "REINFORCE"}, {"env_step": 3072000, "rew": -11.3566, "rew_std": 0.4394, "Agent": "REINFORCE"}, {"env_step": 3102720, "rew": -11.0647, "rew_std": 0.6376, "Agent": "REINFORCE"}, {"env_step": 3133440, "rew": -10.267, "rew_std": 0.5248, "Agent": "REINFORCE"}, {"env_step": 3164160, "rew": -9.4678, "rew_std": 0.6162, "Agent": "REINFORCE"}, {"env_step": 3194880, "rew": -10.6999, "rew_std": 0.5867, "Agent": "REINFORCE"}, {"env_step": 3225600, "rew": -10.2684, "rew_std": 0.8921, "Agent": "REINFORCE"}, {"env_step": 3256320, "rew": -10.4667, "rew_std": 0.734, "Agent": "REINFORCE"}, {"env_step": 3287040, "rew": -10.2591, "rew_std": 0.6946, "Agent": "REINFORCE"}, {"env_step": 3317760, "rew": -10.0482, "rew_std": 0.614, "Agent": "REINFORCE"}, {"env_step": 3348480, "rew": -10.5616, "rew_std": 0.7932, "Agent": "REINFORCE"}, {"env_step": 3379200, "rew": -9.3124, "rew_std": 0.4568, "Agent": "REINFORCE"}, {"env_step": 3409920, "rew": -10.3009, "rew_std": 0.7108, "Agent": "REINFORCE"}, {"env_step": 3440640, "rew": -10.01, "rew_std": 0.4446, "Agent": "REINFORCE"}, {"env_step": 3471360, "rew": -10.0028, "rew_std": 0.7494, "Agent": "REINFORCE"}, {"env_step": 3502080, "rew": -9.7224, "rew_std": 0.3233, "Agent": "REINFORCE"}, {"env_step": 3532800, "rew": -9.9523, "rew_std": 0.4799, "Agent": "REINFORCE"}, {"env_step": 3563520, "rew": -9.4691, "rew_std": 0.3737, "Agent": "REINFORCE"}, {"env_step": 3594240, "rew": -9.7658, "rew_std": 0.3515, "Agent": "REINFORCE"}, {"env_step": 3624960, "rew": -10.0672, "rew_std": 1.0384, "Agent": "REINFORCE"}, {"env_step": 3655680, "rew": -9.4751, "rew_std": 0.6451, "Agent": "REINFORCE"}, {"env_step": 3686400, "rew": -10.7983, "rew_std": 0.5735, "Agent": "REINFORCE"}, {"env_step": 3717120, "rew": -9.8788, "rew_std": 0.5075, "Agent": "REINFORCE"}, {"env_step": 3747840, "rew": -10.0776, "rew_std": 0.455, "Agent": "REINFORCE"}, {"env_step": 3778560, "rew": -9.0511, "rew_std": 0.5551, "Agent": "REINFORCE"}, {"env_step": 3809280, "rew": -10.5304, "rew_std": 0.5492, "Agent": "REINFORCE"}, {"env_step": 3840000, "rew": -10.1546, "rew_std": 1.0191, "Agent": "REINFORCE"}, {"env_step": 3870720, "rew": -9.3241, "rew_std": 1.0923, "Agent": "REINFORCE"}, {"env_step": 3901440, "rew": -8.9206, "rew_std": 0.5791, "Agent": "REINFORCE"}, {"env_step": 3932160, "rew": -9.6375, "rew_std": 0.5193, "Agent": "REINFORCE"}, {"env_step": 3962880, "rew": -9.843, "rew_std": 0.6557, "Agent": "REINFORCE"}, {"env_step": 3993600, "rew": -9.2655, "rew_std": 0.5223, "Agent": "REINFORCE"}, {"env_step": 4024320, "rew": -9.0418, "rew_std": 0.9985, "Agent": "REINFORCE"}, {"env_step": 4055040, "rew": -8.9568, "rew_std": 0.633, "Agent": "REINFORCE"}, {"env_step": 4085760, "rew": -8.8228, "rew_std": 0.5866, "Agent": "REINFORCE"}, {"env_step": 4116480, "rew": -9.051, "rew_std": 0.4739, "Agent": "REINFORCE"}, {"env_step": 4147200, "rew": -9.2519, "rew_std": 0.7584, "Agent": "REINFORCE"}, {"env_step": 4177920, "rew": -9.1386, "rew_std": 0.4962, "Agent": "REINFORCE"}, {"env_step": 4208640, "rew": -9.568, "rew_std": 0.4807, "Agent": "REINFORCE"}, {"env_step": 4239360, "rew": -9.235, "rew_std": 0.6097, "Agent": "REINFORCE"}, {"env_step": 4270080, "rew": -9.2398, "rew_std": 0.9416, "Agent": "REINFORCE"}, {"env_step": 4300800, "rew": -8.7664, "rew_std": 0.5432, "Agent": "REINFORCE"}, {"env_step": 4331520, "rew": -8.7303, "rew_std": 0.6987, "Agent": "REINFORCE"}, {"env_step": 4362240, "rew": -8.3891, "rew_std": 0.5771, "Agent": "REINFORCE"}, {"env_step": 4392960, "rew": -7.6193, "rew_std": 0.6167, "Agent": "REINFORCE"}, {"env_step": 4423680, "rew": -8.6555, "rew_std": 0.5337, "Agent": "REINFORCE"}, {"env_step": 4454400, "rew": -7.9336, "rew_std": 0.4842, "Agent": "REINFORCE"}, {"env_step": 4485120, "rew": -8.0155, "rew_std": 0.5302, "Agent": "REINFORCE"}, {"env_step": 4515840, "rew": -8.5184, "rew_std": 0.3602, "Agent": "REINFORCE"}, {"env_step": 4546560, "rew": -9.7916, "rew_std": 0.4676, "Agent": "REINFORCE"}, {"env_step": 4577280, "rew": -9.0749, "rew_std": 1.0043, "Agent": "REINFORCE"}, {"env_step": 4608000, "rew": -9.0557, "rew_std": 0.4831, "Agent": "REINFORCE"}, {"env_step": 4638720, "rew": -8.2344, "rew_std": 0.4451, "Agent": "REINFORCE"}, {"env_step": 4669440, "rew": -8.4418, "rew_std": 0.5803, "Agent": "REINFORCE"}, {"env_step": 4700160, "rew": -8.3141, "rew_std": 0.398, "Agent": "REINFORCE"}, {"env_step": 4730880, "rew": -8.2198, "rew_std": 0.3204, "Agent": "REINFORCE"}, {"env_step": 4761600, "rew": -9.6126, "rew_std": 0.7645, "Agent": "REINFORCE"}, {"env_step": 4792320, "rew": -8.9173, "rew_std": 0.4035, "Agent": "REINFORCE"}, {"env_step": 4823040, "rew": -7.9342, "rew_std": 0.5715, "Agent": "REINFORCE"}, {"env_step": 4853760, "rew": -8.633, "rew_std": 0.7986, "Agent": "REINFORCE"}, {"env_step": 4884480, "rew": -8.5991, "rew_std": 0.4252, "Agent": "REINFORCE"}, {"env_step": 4915200, "rew": -8.3731, "rew_std": 0.5851, "Agent": "REINFORCE"}, {"env_step": 4945920, "rew": -8.472, "rew_std": 0.6437, "Agent": "REINFORCE"}, {"env_step": 4976640, "rew": -9.0202, "rew_std": 0.568, "Agent": "REINFORCE"}, {"env_step": 5007360, "rew": -7.6257, "rew_std": 0.4051, "Agent": "REINFORCE"}, {"env_step": 5038080, "rew": -7.5025, "rew_std": 0.4499, "Agent": "REINFORCE"}, {"env_step": 5068800, "rew": -7.5077, "rew_std": 0.5008, "Agent": "REINFORCE"}, {"env_step": 5099520, "rew": -8.1766, "rew_std": 0.6425, "Agent": "REINFORCE"}, {"env_step": 5130240, "rew": -7.6867, "rew_std": 0.3841, "Agent": "REINFORCE"}, {"env_step": 5160960, "rew": -7.8887, "rew_std": 0.5593, "Agent": "REINFORCE"}, {"env_step": 5191680, "rew": -7.2022, "rew_std": 0.5648, "Agent": "REINFORCE"}, {"env_step": 5222400, "rew": -8.1184, "rew_std": 0.593, "Agent": "REINFORCE"}, {"env_step": 5253120, "rew": -8.1738, "rew_std": 0.4971, "Agent": "REINFORCE"}, {"env_step": 5283840, "rew": -7.6917, "rew_std": 0.6011, "Agent": "REINFORCE"}, {"env_step": 5314560, "rew": -8.0604, "rew_std": 0.6677, "Agent": "REINFORCE"}, {"env_step": 5345280, "rew": -8.5485, "rew_std": 0.5998, "Agent": "REINFORCE"}, {"env_step": 5376000, "rew": -7.9608, "rew_std": 0.8164, "Agent": "REINFORCE"}, {"env_step": 5406720, "rew": -7.1804, "rew_std": 0.4197, "Agent": "REINFORCE"}, {"env_step": 5437440, "rew": -7.7564, "rew_std": 0.5879, "Agent": "REINFORCE"}, {"env_step": 5468160, "rew": -7.7709, "rew_std": 0.4791, "Agent": "REINFORCE"}, {"env_step": 5498880, "rew": -8.6862, "rew_std": 0.6034, "Agent": "REINFORCE"}, {"env_step": 5529600, "rew": -7.3637, "rew_std": 0.4003, "Agent": "REINFORCE"}, {"env_step": 5560320, "rew": -8.2041, "rew_std": 0.6825, "Agent": "REINFORCE"}, {"env_step": 5591040, "rew": -7.5837, "rew_std": 1.07, "Agent": "REINFORCE"}, {"env_step": 5621760, "rew": -8.0955, "rew_std": 0.5087, "Agent": "REINFORCE"}, {"env_step": 5652480, "rew": -7.9267, "rew_std": 0.3496, "Agent": "REINFORCE"}, {"env_step": 5683200, "rew": -8.4919, "rew_std": 0.5083, "Agent": "REINFORCE"}, {"env_step": 5713920, "rew": -7.7982, "rew_std": 0.6942, "Agent": "REINFORCE"}, {"env_step": 5744640, "rew": -7.5888, "rew_std": 0.814, "Agent": "REINFORCE"}, {"env_step": 5775360, "rew": -7.7097, "rew_std": 0.9339, "Agent": "REINFORCE"}, {"env_step": 5806080, "rew": -8.5273, "rew_std": 0.5199, "Agent": "REINFORCE"}, {"env_step": 5836800, "rew": -7.8432, "rew_std": 0.3993, "Agent": "REINFORCE"}, {"env_step": 5867520, "rew": -6.9768, "rew_std": 0.3155, "Agent": "REINFORCE"}, {"env_step": 5898240, "rew": -8.3769, "rew_std": 0.4494, "Agent": "REINFORCE"}, {"env_step": 5928960, "rew": -7.5693, "rew_std": 0.4086, "Agent": "REINFORCE"}, {"env_step": 5959680, "rew": -7.9647, "rew_std": 0.3701, "Agent": "REINFORCE"}, {"env_step": 5990400, "rew": -8.3405, "rew_std": 0.636, "Agent": "REINFORCE"}, {"env_step": 6021120, "rew": -6.4293, "rew_std": 0.3667, "Agent": "REINFORCE"}, {"env_step": 6051840, "rew": -7.4603, "rew_std": 0.5929, "Agent": "REINFORCE"}, {"env_step": 6082560, "rew": -7.6826, "rew_std": 0.2479, "Agent": "REINFORCE"}, {"env_step": 6113280, "rew": -7.3964, "rew_std": 0.3839, "Agent": "REINFORCE"}, {"env_step": 6144000, "rew": -7.2288, "rew_std": 0.4363, "Agent": "REINFORCE"}, {"env_step": 6174720, "rew": -7.1334, "rew_std": 0.2158, "Agent": "REINFORCE"}, {"env_step": 6205440, "rew": -7.5344, "rew_std": 0.846, "Agent": "REINFORCE"}, {"env_step": 6236160, "rew": -6.912, "rew_std": 0.4014, "Agent": "REINFORCE"}, {"env_step": 6266880, "rew": -7.9701, "rew_std": 0.5672, "Agent": "REINFORCE"}, {"env_step": 6297600, "rew": -7.8618, "rew_std": 0.5215, "Agent": "REINFORCE"}, {"env_step": 6328320, "rew": -8.1969, "rew_std": 0.5516, "Agent": "REINFORCE"}, {"env_step": 6359040, "rew": -8.1615, "rew_std": 0.5294, "Agent": "REINFORCE"}, {"env_step": 6389760, "rew": -7.4611, "rew_std": 0.2738, "Agent": "REINFORCE"}, {"env_step": 6420480, "rew": -7.658, "rew_std": 0.487, "Agent": "REINFORCE"}, {"env_step": 6451200, "rew": -7.0698, "rew_std": 0.3777, "Agent": "REINFORCE"}, {"env_step": 6481920, "rew": -8.3339, "rew_std": 0.3772, "Agent": "REINFORCE"}, {"env_step": 6512640, "rew": -7.4882, "rew_std": 0.7056, "Agent": "REINFORCE"}, {"env_step": 6543360, "rew": -6.911, "rew_std": 0.6282, "Agent": "REINFORCE"}, {"env_step": 6574080, "rew": -7.9438, "rew_std": 0.7116, "Agent": "REINFORCE"}, {"env_step": 6604800, "rew": -7.4463, "rew_std": 0.4285, "Agent": "REINFORCE"}, {"env_step": 6635520, "rew": -7.0292, "rew_std": 0.4409, "Agent": "REINFORCE"}, {"env_step": 6666240, "rew": -7.2936, "rew_std": 0.4047, "Agent": "REINFORCE"}, {"env_step": 6696960, "rew": -7.2766, "rew_std": 0.4944, "Agent": "REINFORCE"}, {"env_step": 6727680, "rew": -7.4212, "rew_std": 0.6075, "Agent": "REINFORCE"}, {"env_step": 6758400, "rew": -6.9896, "rew_std": 0.6367, "Agent": "REINFORCE"}, {"env_step": 6789120, "rew": -6.7185, "rew_std": 0.4203, "Agent": "REINFORCE"}, {"env_step": 6819840, "rew": -7.0371, "rew_std": 0.4686, "Agent": "REINFORCE"}, {"env_step": 6850560, "rew": -7.61, "rew_std": 0.7566, "Agent": "REINFORCE"}, {"env_step": 6881280, "rew": -7.5168, "rew_std": 0.2887, "Agent": "REINFORCE"}, {"env_step": 6912000, "rew": -7.4282, "rew_std": 0.3962, "Agent": "REINFORCE"}, {"env_step": 6942720, "rew": -6.5424, "rew_std": 0.2874, "Agent": "REINFORCE"}, {"env_step": 6973440, "rew": -8.4729, "rew_std": 0.3012, "Agent": "REINFORCE"}, {"env_step": 7004160, "rew": -7.9614, "rew_std": 0.5518, "Agent": "REINFORCE"}, {"env_step": 7034880, "rew": -6.014, "rew_std": 0.3795, "Agent": "REINFORCE"}, {"env_step": 7065600, "rew": -6.993, "rew_std": 0.3695, "Agent": "REINFORCE"}, {"env_step": 7096320, "rew": -7.1158, "rew_std": 0.6192, "Agent": "REINFORCE"}, {"env_step": 7127040, "rew": -7.4814, "rew_std": 0.929, "Agent": "REINFORCE"}, {"env_step": 7157760, "rew": -6.9696, "rew_std": 0.4179, "Agent": "REINFORCE"}, {"env_step": 7188480, "rew": -7.0346, "rew_std": 0.5461, "Agent": "REINFORCE"}, {"env_step": 7219200, "rew": -7.3778, "rew_std": 0.4065, "Agent": "REINFORCE"}, {"env_step": 7249920, "rew": -7.3903, "rew_std": 0.4957, "Agent": "REINFORCE"}, {"env_step": 7280640, "rew": -6.8337, "rew_std": 0.4496, "Agent": "REINFORCE"}, {"env_step": 7311360, "rew": -6.3841, "rew_std": 0.3943, "Agent": "REINFORCE"}, {"env_step": 7342080, "rew": -6.7424, "rew_std": 0.5448, "Agent": "REINFORCE"}, {"env_step": 7372800, "rew": -7.0104, "rew_std": 0.4565, "Agent": "REINFORCE"}, {"env_step": 7403520, "rew": -7.002, "rew_std": 0.7382, "Agent": "REINFORCE"}, {"env_step": 7434240, "rew": -6.6108, "rew_std": 0.4492, "Agent": "REINFORCE"}, {"env_step": 7464960, "rew": -6.5709, "rew_std": 0.6859, "Agent": "REINFORCE"}, {"env_step": 7495680, "rew": -7.881, "rew_std": 0.6181, "Agent": "REINFORCE"}, {"env_step": 7526400, "rew": -7.4604, "rew_std": 0.6759, "Agent": "REINFORCE"}, {"env_step": 7557120, "rew": -6.7943, "rew_std": 0.2675, "Agent": "REINFORCE"}, {"env_step": 7587840, "rew": -6.628, "rew_std": 0.2949, "Agent": "REINFORCE"}, {"env_step": 7618560, "rew": -7.1264, "rew_std": 0.2886, "Agent": "REINFORCE"}, {"env_step": 7649280, "rew": -7.0895, "rew_std": 0.2166, "Agent": "REINFORCE"}, {"env_step": 7680000, "rew": -7.2885, "rew_std": 0.4005, "Agent": "REINFORCE"}, {"env_step": 7710720, "rew": -7.4407, "rew_std": 0.3932, "Agent": "REINFORCE"}, {"env_step": 7741440, "rew": -6.9134, "rew_std": 0.4837, "Agent": "REINFORCE"}, {"env_step": 7772160, "rew": -7.3976, "rew_std": 0.3856, "Agent": "REINFORCE"}, {"env_step": 7802880, "rew": -6.3811, "rew_std": 0.3291, "Agent": "REINFORCE"}, {"env_step": 7833600, "rew": -8.4398, "rew_std": 0.4035, "Agent": "REINFORCE"}, {"env_step": 7864320, "rew": -6.243, "rew_std": 0.3451, "Agent": "REINFORCE"}, {"env_step": 7895040, "rew": -7.9918, "rew_std": 0.5159, "Agent": "REINFORCE"}, {"env_step": 7925760, "rew": -7.4007, "rew_std": 0.6032, "Agent": "REINFORCE"}, {"env_step": 7956480, "rew": -7.2012, "rew_std": 0.3187, "Agent": "REINFORCE"}, {"env_step": 7987200, "rew": -7.2638, "rew_std": 0.2397, "Agent": "REINFORCE"}, {"env_step": 8017920, "rew": -6.9558, "rew_std": 0.2604, "Agent": "REINFORCE"}, {"env_step": 8048640, "rew": -7.0604, "rew_std": 0.4733, "Agent": "REINFORCE"}, {"env_step": 8079360, "rew": -6.3747, "rew_std": 0.3021, "Agent": "REINFORCE"}, {"env_step": 8110080, "rew": -6.4015, "rew_std": 0.2167, "Agent": "REINFORCE"}, {"env_step": 8140800, "rew": -7.0195, "rew_std": 0.5581, "Agent": "REINFORCE"}, {"env_step": 8171520, "rew": -6.7174, "rew_std": 0.2973, "Agent": "REINFORCE"}, {"env_step": 8202240, "rew": -6.6271, "rew_std": 0.2493, "Agent": "REINFORCE"}, {"env_step": 8232960, "rew": -6.8147, "rew_std": 0.4268, "Agent": "REINFORCE"}, {"env_step": 8263680, "rew": -6.5861, "rew_std": 0.2724, "Agent": "REINFORCE"}, {"env_step": 8294400, "rew": -7.1314, "rew_std": 0.3291, "Agent": "REINFORCE"}, {"env_step": 8325120, "rew": -6.4392, "rew_std": 0.4481, "Agent": "REINFORCE"}, {"env_step": 8355840, "rew": -6.304, "rew_std": 0.2563, "Agent": "REINFORCE"}, {"env_step": 8386560, "rew": -5.7995, "rew_std": 0.3314, "Agent": "REINFORCE"}, {"env_step": 8417280, "rew": -7.7415, "rew_std": 0.2744, "Agent": "REINFORCE"}, {"env_step": 8448000, "rew": -6.5171, "rew_std": 0.2879, "Agent": "REINFORCE"}, {"env_step": 8478720, "rew": -6.803, "rew_std": 0.4621, "Agent": "REINFORCE"}, {"env_step": 8509440, "rew": -7.1545, "rew_std": 0.4238, "Agent": "REINFORCE"}, {"env_step": 8540160, "rew": -7.0141, "rew_std": 0.5667, "Agent": "REINFORCE"}, {"env_step": 8570880, "rew": -7.498, "rew_std": 0.572, "Agent": "REINFORCE"}, {"env_step": 8601600, "rew": -6.8268, "rew_std": 0.1592, "Agent": "REINFORCE"}, {"env_step": 8632320, "rew": -7.1468, "rew_std": 1.0048, "Agent": "REINFORCE"}, {"env_step": 8663040, "rew": -6.231, "rew_std": 0.4246, "Agent": "REINFORCE"}, {"env_step": 8693760, "rew": -6.7758, "rew_std": 0.2444, "Agent": "REINFORCE"}, {"env_step": 8724480, "rew": -6.8614, "rew_std": 0.3005, "Agent": "REINFORCE"}, {"env_step": 8755200, "rew": -6.4715, "rew_std": 0.5622, "Agent": "REINFORCE"}, {"env_step": 8785920, "rew": -7.538, "rew_std": 0.33, "Agent": "REINFORCE"}, {"env_step": 8816640, "rew": -6.9378, "rew_std": 0.441, "Agent": "REINFORCE"}, {"env_step": 8847360, "rew": -7.0213, "rew_std": 0.4161, "Agent": "REINFORCE"}, {"env_step": 8878080, "rew": -6.3532, "rew_std": 0.2787, "Agent": "REINFORCE"}, {"env_step": 8908800, "rew": -5.9954, "rew_std": 0.4256, "Agent": "REINFORCE"}, {"env_step": 8939520, "rew": -6.4347, "rew_std": 0.337, "Agent": "REINFORCE"}, {"env_step": 8970240, "rew": -6.7357, "rew_std": 0.2993, "Agent": "REINFORCE"}, {"env_step": 9000960, "rew": -7.0348, "rew_std": 0.2923, "Agent": "REINFORCE"}, {"env_step": 9031680, "rew": -7.2234, "rew_std": 0.3924, "Agent": "REINFORCE"}, {"env_step": 9062400, "rew": -6.7263, "rew_std": 0.4882, "Agent": "REINFORCE"}, {"env_step": 9093120, "rew": -5.9239, "rew_std": 0.3753, "Agent": "REINFORCE"}, {"env_step": 9123840, "rew": -7.0764, "rew_std": 0.5975, "Agent": "REINFORCE"}, {"env_step": 9154560, "rew": -7.4387, "rew_std": 0.4296, "Agent": "REINFORCE"}, {"env_step": 9185280, "rew": -6.1808, "rew_std": 0.3834, "Agent": "REINFORCE"}, {"env_step": 9216000, "rew": -7.0908, "rew_std": 0.3954, "Agent": "REINFORCE"}, {"env_step": 9246720, "rew": -6.3499, "rew_std": 0.3025, "Agent": "REINFORCE"}, {"env_step": 9277440, "rew": -6.7003, "rew_std": 0.4033, "Agent": "REINFORCE"}, {"env_step": 9308160, "rew": -6.1223, "rew_std": 0.3594, "Agent": "REINFORCE"}, {"env_step": 9338880, "rew": -5.547, "rew_std": 0.2432, "Agent": "REINFORCE"}, {"env_step": 9369600, "rew": -6.7066, "rew_std": 0.2806, "Agent": "REINFORCE"}, {"env_step": 9400320, "rew": -6.4505, "rew_std": 0.4009, "Agent": "REINFORCE"}, {"env_step": 9431040, "rew": -6.6948, "rew_std": 0.4809, "Agent": "REINFORCE"}, {"env_step": 9461760, "rew": -6.4916, "rew_std": 0.2387, "Agent": "REINFORCE"}, {"env_step": 9492480, "rew": -6.337, "rew_std": 0.3624, "Agent": "REINFORCE"}, {"env_step": 9523200, "rew": -6.3873, "rew_std": 0.3239, "Agent": "REINFORCE"}, {"env_step": 9553920, "rew": -7.3367, "rew_std": 0.275, "Agent": "REINFORCE"}, {"env_step": 9584640, "rew": -6.1671, "rew_std": 0.3965, "Agent": "REINFORCE"}, {"env_step": 9615360, "rew": -6.2706, "rew_std": 0.3238, "Agent": "REINFORCE"}, {"env_step": 9646080, "rew": -6.1982, "rew_std": 0.5299, "Agent": "REINFORCE"}, {"env_step": 9676800, "rew": -6.5887, "rew_std": 0.4414, "Agent": "REINFORCE"}, {"env_step": 9707520, "rew": -7.471, "rew_std": 0.6133, "Agent": "REINFORCE"}, {"env_step": 9738240, "rew": -6.9098, "rew_std": 0.4671, "Agent": "REINFORCE"}, {"env_step": 9768960, "rew": -6.9614, "rew_std": 0.2386, "Agent": "REINFORCE"}, {"env_step": 9799680, "rew": -6.2465, "rew_std": 0.2412, "Agent": "REINFORCE"}, {"env_step": 9830400, "rew": -6.2565, "rew_std": 0.1699, "Agent": "REINFORCE"}, {"env_step": 9861120, "rew": -6.6373, "rew_std": 0.3563, "Agent": "REINFORCE"}, {"env_step": 9891840, "rew": -6.426, "rew_std": 0.3518, "Agent": "REINFORCE"}, {"env_step": 9922560, "rew": -6.5556, "rew_std": 0.3843, "Agent": "REINFORCE"}, {"env_step": 9953280, "rew": -6.0887, "rew_std": 0.323, "Agent": "REINFORCE"}, {"env_step": 9984000, "rew": -6.5108, "rew_std": 0.5198, "Agent": "REINFORCE"}, {"env_step": 10014720, "rew": -6.1655, "rew_std": 0.4087, "Agent": "REINFORCE"}, {"env_step": 10045440, "rew": -6.6196, "rew_std": 0.4188, "Agent": "REINFORCE"}, {"env_step": 10076160, "rew": -6.2191, "rew_std": 0.3846, "Agent": "REINFORCE"}, {"env_step": 10106880, "rew": -7.0509, "rew_std": 0.4208, "Agent": "REINFORCE"}, {"env_step": 10137600, "rew": -6.7348, "rew_std": 0.3687, "Agent": "REINFORCE"}, {"env_step": 10168320, "rew": -6.9149, "rew_std": 0.2471, "Agent": "REINFORCE"}, {"env_step": 10199040, "rew": -6.6179, "rew_std": 0.3684, "Agent": "REINFORCE"}, {"env_step": 10229760, "rew": -6.5823, "rew_std": 0.2693, "Agent": "REINFORCE"}, {"env_step": 0, "rew": -39.7236, "rew_std": 1.6682, "Agent": "A2C"}, {"env_step": 30000, "rew": -49.5126, "rew_std": 6.7796, "Agent": "A2C"}, {"env_step": 60000, "rew": -47.0013, "rew_std": 14.3514, "Agent": "A2C"}, {"env_step": 90000, "rew": -35.2423, "rew_std": 16.0327, "Agent": "A2C"}, {"env_step": 120000, "rew": -27.4804, "rew_std": 15.9979, "Agent": "A2C"}, {"env_step": 150000, "rew": -23.4895, "rew_std": 16.217, "Agent": "A2C"}, {"env_step": 180000, "rew": -21.7784, "rew_std": 15.5391, "Agent": "A2C"}, {"env_step": 210000, "rew": -19.4302, "rew_std": 14.4032, "Agent": "A2C"}, {"env_step": 240000, "rew": -17.3468, "rew_std": 15.2334, "Agent": "A2C"}, {"env_step": 270000, "rew": -16.9129, "rew_std": 16.6739, "Agent": "A2C"}, {"env_step": 300000, "rew": -15.4613, "rew_std": 16.6621, "Agent": "A2C"}, {"env_step": 330000, "rew": -15.2061, "rew_std": 15.9173, "Agent": "A2C"}, {"env_step": 360000, "rew": -15.8345, "rew_std": 15.892, "Agent": "A2C"}, {"env_step": 390000, "rew": -12.0236, "rew_std": 8.8214, "Agent": "A2C"}, {"env_step": 420000, "rew": -10.2533, "rew_std": 3.8797, "Agent": "A2C"}, {"env_step": 450000, "rew": -9.3766, "rew_std": 2.3657, "Agent": "A2C"}, {"env_step": 480000, "rew": -8.8854, "rew_std": 2.9041, "Agent": "A2C"}, {"env_step": 510000, "rew": -9.0523, "rew_std": 1.5466, "Agent": "A2C"}, {"env_step": 540000, "rew": -9.4109, "rew_std": 1.291, "Agent": "A2C"}, {"env_step": 570000, "rew": -9.4005, "rew_std": 0.9042, "Agent": "A2C"}, {"env_step": 600000, "rew": -9.0146, "rew_std": 1.0466, "Agent": "A2C"}, {"env_step": 630000, "rew": -8.1351, "rew_std": 0.9026, "Agent": "A2C"}, {"env_step": 660000, "rew": -9.152, "rew_std": 0.8241, "Agent": "A2C"}, {"env_step": 690000, "rew": -8.4558, "rew_std": 0.9946, "Agent": "A2C"}, {"env_step": 720000, "rew": -7.3382, "rew_std": 1.0027, "Agent": "A2C"}, {"env_step": 750000, "rew": -7.5766, "rew_std": 1.0436, "Agent": "A2C"}, {"env_step": 780000, "rew": -7.7202, "rew_std": 0.8769, "Agent": "A2C"}, {"env_step": 810000, "rew": -7.1812, "rew_std": 0.9643, "Agent": "A2C"}, {"env_step": 840000, "rew": -7.4013, "rew_std": 0.8819, "Agent": "A2C"}, {"env_step": 870000, "rew": -7.1151, "rew_std": 0.3934, "Agent": "A2C"}, {"env_step": 900000, "rew": -7.1768, "rew_std": 0.9076, "Agent": "A2C"}, {"env_step": 930000, "rew": -6.7395, "rew_std": 2.2945, "Agent": "A2C"}, {"env_step": 960000, "rew": -6.9028, "rew_std": 1.674, "Agent": "A2C"}, {"env_step": 990000, "rew": -7.5811, "rew_std": 0.7885, "Agent": "A2C"}, {"env_step": 1020000, "rew": -6.0354, "rew_std": 0.527, "Agent": "A2C"}, {"env_step": 1050000, "rew": -6.4296, "rew_std": 0.9365, "Agent": "A2C"}, {"env_step": 1080000, "rew": -6.5335, "rew_std": 0.7245, "Agent": "A2C"}, {"env_step": 1110000, "rew": -6.6956, "rew_std": 1.1084, "Agent": "A2C"}, {"env_step": 1140000, "rew": -6.7058, "rew_std": 0.8603, "Agent": "A2C"}, {"env_step": 1170000, "rew": -6.4484, "rew_std": 0.6742, "Agent": "A2C"}, {"env_step": 1200000, "rew": -6.7887, "rew_std": 0.8626, "Agent": "A2C"}, {"env_step": 1230000, "rew": -7.0301, "rew_std": 0.8594, "Agent": "A2C"}, {"env_step": 1260000, "rew": -6.4574, "rew_std": 1.0401, "Agent": "A2C"}, {"env_step": 1290000, "rew": -7.4859, "rew_std": 2.1446, "Agent": "A2C"}, {"env_step": 1320000, "rew": -5.9742, "rew_std": 0.61, "Agent": "A2C"}, {"env_step": 1350000, "rew": -5.7317, "rew_std": 0.5252, "Agent": "A2C"}, {"env_step": 1380000, "rew": -5.8109, "rew_std": 0.6197, "Agent": "A2C"}, {"env_step": 1410000, "rew": -5.2524, "rew_std": 0.614, "Agent": "A2C"}, {"env_step": 1440000, "rew": -6.1422, "rew_std": 0.8794, "Agent": "A2C"}, {"env_step": 1470000, "rew": -6.0556, "rew_std": 0.5942, "Agent": "A2C"}, {"env_step": 1500000, "rew": -5.9744, "rew_std": 0.613, "Agent": "A2C"}, {"env_step": 1530000, "rew": -6.3628, "rew_std": 0.7473, "Agent": "A2C"}, {"env_step": 1560000, "rew": -6.719, "rew_std": 0.8635, "Agent": "A2C"}, {"env_step": 1590000, "rew": -6.3226, "rew_std": 0.4271, "Agent": "A2C"}, {"env_step": 1620000, "rew": -5.7314, "rew_std": 0.6983, "Agent": "A2C"}, {"env_step": 1650000, "rew": -6.3741, "rew_std": 0.6323, "Agent": "A2C"}, {"env_step": 1680000, "rew": -5.584, "rew_std": 1.0605, "Agent": "A2C"}, {"env_step": 1710000, "rew": -6.4225, "rew_std": 0.5278, "Agent": "A2C"}, {"env_step": 1740000, "rew": -5.7861, "rew_std": 0.5744, "Agent": "A2C"}, {"env_step": 1770000, "rew": -6.1038, "rew_std": 0.487, "Agent": "A2C"}, {"env_step": 1800000, "rew": -6.1468, "rew_std": 0.3817, "Agent": "A2C"}, {"env_step": 1830000, "rew": -6.3976, "rew_std": 0.4491, "Agent": "A2C"}, {"env_step": 1860000, "rew": -5.5454, "rew_std": 0.6492, "Agent": "A2C"}, {"env_step": 1890000, "rew": -5.3233, "rew_std": 0.5404, "Agent": "A2C"}, {"env_step": 1920000, "rew": -6.5189, "rew_std": 0.7128, "Agent": "A2C"}, {"env_step": 1950000, "rew": -5.5919, "rew_std": 0.7504, "Agent": "A2C"}, {"env_step": 1980000, "rew": -5.8653, "rew_std": 0.5621, "Agent": "A2C"}, {"env_step": 2010000, "rew": -6.7014, "rew_std": 0.5326, "Agent": "A2C"}, {"env_step": 2040000, "rew": -6.2351, "rew_std": 0.592, "Agent": "A2C"}, {"env_step": 2070000, "rew": -6.1591, "rew_std": 0.7988, "Agent": "A2C"}, {"env_step": 2100000, "rew": -6.4472, "rew_std": 0.6605, "Agent": "A2C"}, {"env_step": 2130000, "rew": -6.7471, "rew_std": 0.5177, "Agent": "A2C"}, {"env_step": 2160000, "rew": -5.8213, "rew_std": 0.7378, "Agent": "A2C"}, {"env_step": 2190000, "rew": -6.4161, "rew_std": 0.7222, "Agent": "A2C"}, {"env_step": 2220000, "rew": -5.8552, "rew_std": 0.5859, "Agent": "A2C"}, {"env_step": 2250000, "rew": -6.6032, "rew_std": 0.6478, "Agent": "A2C"}, {"env_step": 2280000, "rew": -6.1205, "rew_std": 0.4762, "Agent": "A2C"}, {"env_step": 2310000, "rew": -5.4231, "rew_std": 0.5739, "Agent": "A2C"}, {"env_step": 2340000, "rew": -6.4015, "rew_std": 0.7271, "Agent": "A2C"}, {"env_step": 2370000, "rew": -5.4117, "rew_std": 0.6904, "Agent": "A2C"}, {"env_step": 2400000, "rew": -6.2049, "rew_std": 0.6572, "Agent": "A2C"}, {"env_step": 2430000, "rew": -7.0885, "rew_std": 0.6712, "Agent": "A2C"}, {"env_step": 2460000, "rew": -5.7508, "rew_std": 0.7739, "Agent": "A2C"}, {"env_step": 2490000, "rew": -5.8503, "rew_std": 0.7646, "Agent": "A2C"}, {"env_step": 2520000, "rew": -6.5471, "rew_std": 0.7317, "Agent": "A2C"}, {"env_step": 2550000, "rew": -5.634, "rew_std": 0.5131, "Agent": "A2C"}, {"env_step": 2580000, "rew": -5.234, "rew_std": 0.5039, "Agent": "A2C"}, {"env_step": 2610000, "rew": -6.1475, "rew_std": 0.7397, "Agent": "A2C"}, {"env_step": 2640000, "rew": -5.426, "rew_std": 0.5692, "Agent": "A2C"}, {"env_step": 2670000, "rew": -5.9137, "rew_std": 0.9543, "Agent": "A2C"}, {"env_step": 2700000, "rew": -6.4246, "rew_std": 0.8012, "Agent": "A2C"}, {"env_step": 2730000, "rew": -5.8467, "rew_std": 0.5215, "Agent": "A2C"}, {"env_step": 2760000, "rew": -5.6174, "rew_std": 0.4854, "Agent": "A2C"}, {"env_step": 2790000, "rew": -6.5128, "rew_std": 0.5292, "Agent": "A2C"}, {"env_step": 2820000, "rew": -6.1005, "rew_std": 0.7479, "Agent": "A2C"}, {"env_step": 2850000, "rew": -5.9128, "rew_std": 0.6469, "Agent": "A2C"}, {"env_step": 2880000, "rew": -5.6554, "rew_std": 0.6749, "Agent": "A2C"}, {"env_step": 2910000, "rew": -6.1043, "rew_std": 0.508, "Agent": "A2C"}, {"env_step": 2940000, "rew": -5.8506, "rew_std": 0.6609, "Agent": "A2C"}, {"env_step": 2970000, "rew": -6.5352, "rew_std": 0.4961, "Agent": "A2C"}, {"env_step": 3000000, "rew": -6.2979, "rew_std": 0.5372, "Agent": "A2C"}] \ No newline at end of file diff --git a/examples/mujoco/benchmark/Swimmer-v3/result.json b/examples/mujoco/benchmark/Swimmer-v3/result.json new file mode 100644 index 0000000..a14ccbc --- /dev/null +++ b/examples/mujoco/benchmark/Swimmer-v3/result.json @@ -0,0 +1 @@ +[{"env_step": 0, "rew": -7.7439, "rew_std": 16.8878, "Agent": "SAC"}, {"env_step": 5000, "rew": 26.2874, "rew_std": 10.5409, "Agent": "SAC"}, {"env_step": 10000, "rew": 25.6709, "rew_std": 8.9848, "Agent": "SAC"}, {"env_step": 15000, "rew": 33.9519, "rew_std": 3.7954, "Agent": "SAC"}, {"env_step": 20000, "rew": 38.1491, "rew_std": 2.543, "Agent": "SAC"}, {"env_step": 25000, "rew": 39.3281, "rew_std": 2.1892, "Agent": "SAC"}, {"env_step": 30000, "rew": 38.0972, "rew_std": 2.6193, "Agent": "SAC"}, {"env_step": 35000, "rew": 39.4004, "rew_std": 1.9991, "Agent": "SAC"}, {"env_step": 40000, "rew": 39.3137, "rew_std": 1.7931, "Agent": "SAC"}, {"env_step": 45000, "rew": 38.8152, "rew_std": 1.6557, "Agent": "SAC"}, {"env_step": 50000, "rew": 39.8585, "rew_std": 1.157, "Agent": "SAC"}, {"env_step": 55000, "rew": 38.1781, "rew_std": 4.3822, "Agent": "SAC"}, {"env_step": 60000, "rew": 37.8314, "rew_std": 4.4601, "Agent": "SAC"}, {"env_step": 65000, "rew": 38.3241, "rew_std": 2.8926, "Agent": "SAC"}, {"env_step": 70000, "rew": 39.395, "rew_std": 1.2904, "Agent": "SAC"}, {"env_step": 75000, "rew": 39.6681, "rew_std": 1.354, "Agent": "SAC"}, {"env_step": 80000, "rew": 40.2258, "rew_std": 1.3807, "Agent": "SAC"}, {"env_step": 85000, "rew": 40.0252, "rew_std": 1.7095, "Agent": "SAC"}, {"env_step": 90000, "rew": 39.8833, "rew_std": 1.071, "Agent": "SAC"}, {"env_step": 95000, "rew": 37.6892, "rew_std": 4.3594, "Agent": "SAC"}, {"env_step": 100000, "rew": 39.866, "rew_std": 1.494, "Agent": "SAC"}, {"env_step": 105000, "rew": 40.3161, "rew_std": 1.5569, "Agent": "SAC"}, {"env_step": 110000, "rew": 38.6797, "rew_std": 5.4645, "Agent": "SAC"}, {"env_step": 115000, "rew": 39.4061, "rew_std": 1.0227, "Agent": "SAC"}, {"env_step": 120000, "rew": 39.0564, "rew_std": 1.6067, "Agent": "SAC"}, {"env_step": 125000, "rew": 41.2194, "rew_std": 1.1145, "Agent": "SAC"}, {"env_step": 130000, "rew": 38.7546, "rew_std": 2.59, "Agent": "SAC"}, {"env_step": 135000, "rew": 41.1758, "rew_std": 1.2852, "Agent": "SAC"}, {"env_step": 140000, "rew": 40.7014, "rew_std": 1.3486, "Agent": "SAC"}, {"env_step": 145000, "rew": 40.0147, "rew_std": 1.7605, "Agent": "SAC"}, {"env_step": 150000, "rew": 40.3221, "rew_std": 1.2607, "Agent": "SAC"}, {"env_step": 155000, "rew": 41.6201, "rew_std": 1.3049, "Agent": "SAC"}, {"env_step": 160000, "rew": 41.3052, "rew_std": 1.0019, "Agent": "SAC"}, {"env_step": 165000, "rew": 39.9204, "rew_std": 2.8623, "Agent": "SAC"}, {"env_step": 170000, "rew": 40.9499, "rew_std": 1.6341, "Agent": "SAC"}, {"env_step": 175000, "rew": 42.1303, "rew_std": 1.635, "Agent": "SAC"}, {"env_step": 180000, "rew": 41.3798, "rew_std": 1.3728, "Agent": "SAC"}, {"env_step": 185000, "rew": 41.6402, "rew_std": 1.3664, "Agent": "SAC"}, {"env_step": 190000, "rew": 41.8703, "rew_std": 1.0121, "Agent": "SAC"}, {"env_step": 195000, "rew": 41.686, "rew_std": 2.8906, "Agent": "SAC"}, {"env_step": 200000, "rew": 41.7817, "rew_std": 0.8474, "Agent": "SAC"}, {"env_step": 205000, "rew": 41.7951, "rew_std": 1.7754, "Agent": "SAC"}, {"env_step": 210000, "rew": 42.0247, "rew_std": 1.4268, "Agent": "SAC"}, {"env_step": 215000, "rew": 42.3229, "rew_std": 0.7668, "Agent": "SAC"}, {"env_step": 220000, "rew": 40.6762, "rew_std": 2.6318, "Agent": "SAC"}, {"env_step": 225000, "rew": 41.497, "rew_std": 0.6603, "Agent": "SAC"}, {"env_step": 230000, "rew": 41.5852, "rew_std": 1.2995, "Agent": "SAC"}, {"env_step": 235000, "rew": 39.668, "rew_std": 6.5793, "Agent": "SAC"}, {"env_step": 240000, "rew": 41.4187, "rew_std": 0.7364, "Agent": "SAC"}, {"env_step": 245000, "rew": 41.6194, "rew_std": 0.9224, "Agent": "SAC"}, {"env_step": 250000, "rew": 42.5275, "rew_std": 0.8569, "Agent": "SAC"}, {"env_step": 255000, "rew": 42.1773, "rew_std": 0.8542, "Agent": "SAC"}, {"env_step": 260000, "rew": 40.6498, "rew_std": 1.9453, "Agent": "SAC"}, {"env_step": 265000, "rew": 41.8543, "rew_std": 0.7636, "Agent": "SAC"}, {"env_step": 270000, "rew": 41.4246, "rew_std": 1.4716, "Agent": "SAC"}, {"env_step": 275000, "rew": 41.6244, "rew_std": 1.4436, "Agent": "SAC"}, {"env_step": 280000, "rew": 41.762, "rew_std": 1.3608, "Agent": "SAC"}, {"env_step": 285000, "rew": 41.7634, "rew_std": 2.4729, "Agent": "SAC"}, {"env_step": 290000, "rew": 42.5842, "rew_std": 1.3938, "Agent": "SAC"}, {"env_step": 295000, "rew": 42.0463, "rew_std": 1.3349, "Agent": "SAC"}, {"env_step": 300000, "rew": 41.4114, "rew_std": 2.0244, "Agent": "SAC"}, {"env_step": 305000, "rew": 41.8277, "rew_std": 0.8916, "Agent": "SAC"}, {"env_step": 310000, "rew": 42.9752, "rew_std": 1.0657, "Agent": "SAC"}, {"env_step": 315000, "rew": 42.2197, "rew_std": 0.99, "Agent": "SAC"}, {"env_step": 320000, "rew": 41.3587, "rew_std": 1.7308, "Agent": "SAC"}, {"env_step": 325000, "rew": 42.1496, "rew_std": 0.8021, "Agent": "SAC"}, {"env_step": 330000, "rew": 42.0622, "rew_std": 1.483, "Agent": "SAC"}, {"env_step": 335000, "rew": 42.4782, "rew_std": 1.7003, "Agent": "SAC"}, {"env_step": 340000, "rew": 42.1533, "rew_std": 0.7591, "Agent": "SAC"}, {"env_step": 345000, "rew": 41.324, "rew_std": 1.244, "Agent": "SAC"}, {"env_step": 350000, "rew": 42.606, "rew_std": 1.386, "Agent": "SAC"}, {"env_step": 355000, "rew": 41.5811, "rew_std": 1.7395, "Agent": "SAC"}, {"env_step": 360000, "rew": 42.418, "rew_std": 1.5588, "Agent": "SAC"}, {"env_step": 365000, "rew": 41.6303, "rew_std": 0.759, "Agent": "SAC"}, {"env_step": 370000, "rew": 42.2691, "rew_std": 0.9922, "Agent": "SAC"}, {"env_step": 375000, "rew": 42.5791, "rew_std": 1.3288, "Agent": "SAC"}, {"env_step": 380000, "rew": 42.7526, "rew_std": 1.0351, "Agent": "SAC"}, {"env_step": 385000, "rew": 43.2361, "rew_std": 1.4465, "Agent": "SAC"}, {"env_step": 390000, "rew": 41.8614, "rew_std": 0.9324, "Agent": "SAC"}, {"env_step": 395000, "rew": 42.3396, "rew_std": 1.3756, "Agent": "SAC"}, {"env_step": 400000, "rew": 41.9191, "rew_std": 1.1385, "Agent": "SAC"}, {"env_step": 405000, "rew": 41.5898, "rew_std": 1.1394, "Agent": "SAC"}, {"env_step": 410000, "rew": 41.2049, "rew_std": 2.1405, "Agent": "SAC"}, {"env_step": 415000, "rew": 43.1547, "rew_std": 1.3899, "Agent": "SAC"}, {"env_step": 420000, "rew": 41.9552, "rew_std": 1.4365, "Agent": "SAC"}, {"env_step": 425000, "rew": 42.865, "rew_std": 0.7121, "Agent": "SAC"}, {"env_step": 430000, "rew": 43.2182, "rew_std": 1.2923, "Agent": "SAC"}, {"env_step": 435000, "rew": 43.0546, "rew_std": 1.1176, "Agent": "SAC"}, {"env_step": 440000, "rew": 42.1793, "rew_std": 1.0319, "Agent": "SAC"}, {"env_step": 445000, "rew": 42.3744, "rew_std": 1.1834, "Agent": "SAC"}, {"env_step": 450000, "rew": 42.7055, "rew_std": 1.0265, "Agent": "SAC"}, {"env_step": 455000, "rew": 42.7255, "rew_std": 0.888, "Agent": "SAC"}, {"env_step": 460000, "rew": 42.2179, "rew_std": 1.6927, "Agent": "SAC"}, {"env_step": 465000, "rew": 42.4877, "rew_std": 1.298, "Agent": "SAC"}, {"env_step": 470000, "rew": 42.8837, "rew_std": 1.0435, "Agent": "SAC"}, {"env_step": 475000, "rew": 42.9676, "rew_std": 0.9981, "Agent": "SAC"}, {"env_step": 480000, "rew": 42.9281, "rew_std": 1.965, "Agent": "SAC"}, {"env_step": 485000, "rew": 42.3553, "rew_std": 0.884, "Agent": "SAC"}, {"env_step": 490000, "rew": 42.5956, "rew_std": 0.8306, "Agent": "SAC"}, {"env_step": 495000, "rew": 42.3804, "rew_std": 1.8631, "Agent": "SAC"}, {"env_step": 500000, "rew": 40.6518, "rew_std": 2.8328, "Agent": "SAC"}, {"env_step": 505000, "rew": 41.8868, "rew_std": 2.1089, "Agent": "SAC"}, {"env_step": 510000, "rew": 42.2984, "rew_std": 1.8482, "Agent": "SAC"}, {"env_step": 515000, "rew": 42.5435, "rew_std": 1.5197, "Agent": "SAC"}, {"env_step": 520000, "rew": 42.5111, "rew_std": 1.2591, "Agent": "SAC"}, {"env_step": 525000, "rew": 42.5548, "rew_std": 1.581, "Agent": "SAC"}, {"env_step": 530000, "rew": 43.2509, "rew_std": 1.0975, "Agent": "SAC"}, {"env_step": 535000, "rew": 42.8992, "rew_std": 0.6006, "Agent": "SAC"}, {"env_step": 540000, "rew": 42.6448, "rew_std": 0.6434, "Agent": "SAC"}, {"env_step": 545000, "rew": 43.6739, "rew_std": 1.03, "Agent": "SAC"}, {"env_step": 550000, "rew": 42.1373, "rew_std": 1.1803, "Agent": "SAC"}, {"env_step": 555000, "rew": 43.392, "rew_std": 0.7966, "Agent": "SAC"}, {"env_step": 560000, "rew": 42.998, "rew_std": 1.7691, "Agent": "SAC"}, {"env_step": 565000, "rew": 42.3194, "rew_std": 1.6072, "Agent": "SAC"}, {"env_step": 570000, "rew": 42.4264, "rew_std": 1.0855, "Agent": "SAC"}, {"env_step": 575000, "rew": 43.0293, "rew_std": 1.5761, "Agent": "SAC"}, {"env_step": 580000, "rew": 43.1212, "rew_std": 1.0842, "Agent": "SAC"}, {"env_step": 585000, "rew": 43.7142, "rew_std": 0.8072, "Agent": "SAC"}, {"env_step": 590000, "rew": 42.9435, "rew_std": 0.8323, "Agent": "SAC"}, {"env_step": 595000, "rew": 43.0102, "rew_std": 1.1116, "Agent": "SAC"}, {"env_step": 600000, "rew": 42.6918, "rew_std": 1.7408, "Agent": "SAC"}, {"env_step": 605000, "rew": 42.9151, "rew_std": 1.5017, "Agent": "SAC"}, {"env_step": 610000, "rew": 42.4029, "rew_std": 1.1383, "Agent": "SAC"}, {"env_step": 615000, "rew": 42.2635, "rew_std": 1.1714, "Agent": "SAC"}, {"env_step": 620000, "rew": 42.2494, "rew_std": 2.1549, "Agent": "SAC"}, {"env_step": 625000, "rew": 42.9938, "rew_std": 0.8821, "Agent": "SAC"}, {"env_step": 630000, "rew": 42.7989, "rew_std": 0.8944, "Agent": "SAC"}, {"env_step": 635000, "rew": 42.8052, "rew_std": 1.5786, "Agent": "SAC"}, {"env_step": 640000, "rew": 42.9076, "rew_std": 0.7705, "Agent": "SAC"}, {"env_step": 645000, "rew": 42.7719, "rew_std": 0.8277, "Agent": "SAC"}, {"env_step": 650000, "rew": 42.0232, "rew_std": 1.5944, "Agent": "SAC"}, {"env_step": 655000, "rew": 42.4545, "rew_std": 1.5504, "Agent": "SAC"}, {"env_step": 660000, "rew": 42.6647, "rew_std": 1.2738, "Agent": "SAC"}, {"env_step": 665000, "rew": 42.8804, "rew_std": 0.8634, "Agent": "SAC"}, {"env_step": 670000, "rew": 42.26, "rew_std": 1.2206, "Agent": "SAC"}, {"env_step": 675000, "rew": 42.8885, "rew_std": 0.7465, "Agent": "SAC"}, {"env_step": 680000, "rew": 43.2638, "rew_std": 0.5848, "Agent": "SAC"}, {"env_step": 685000, "rew": 42.8161, "rew_std": 1.0889, "Agent": "SAC"}, {"env_step": 690000, "rew": 43.6413, "rew_std": 0.8257, "Agent": "SAC"}, {"env_step": 695000, "rew": 42.0686, "rew_std": 2.5861, "Agent": "SAC"}, {"env_step": 700000, "rew": 42.6375, "rew_std": 0.859, "Agent": "SAC"}, {"env_step": 705000, "rew": 43.1986, "rew_std": 0.924, "Agent": "SAC"}, {"env_step": 710000, "rew": 42.8405, "rew_std": 1.2386, "Agent": "SAC"}, {"env_step": 715000, "rew": 42.6156, "rew_std": 2.2722, "Agent": "SAC"}, {"env_step": 720000, "rew": 42.7577, "rew_std": 0.8786, "Agent": "SAC"}, {"env_step": 725000, "rew": 43.4543, "rew_std": 1.2874, "Agent": "SAC"}, {"env_step": 730000, "rew": 43.7171, "rew_std": 0.6439, "Agent": "SAC"}, {"env_step": 735000, "rew": 42.5117, "rew_std": 1.2804, "Agent": "SAC"}, {"env_step": 740000, "rew": 43.3573, "rew_std": 1.2178, "Agent": "SAC"}, {"env_step": 745000, "rew": 42.8615, "rew_std": 1.4943, "Agent": "SAC"}, {"env_step": 750000, "rew": 43.3079, "rew_std": 1.2424, "Agent": "SAC"}, {"env_step": 755000, "rew": 42.3746, "rew_std": 1.0121, "Agent": "SAC"}, {"env_step": 760000, "rew": 42.8132, "rew_std": 1.0824, "Agent": "SAC"}, {"env_step": 765000, "rew": 43.7006, "rew_std": 0.7133, "Agent": "SAC"}, {"env_step": 770000, "rew": 43.3221, "rew_std": 0.9818, "Agent": "SAC"}, {"env_step": 775000, "rew": 42.4514, "rew_std": 1.4063, "Agent": "SAC"}, {"env_step": 780000, "rew": 43.8367, "rew_std": 0.9614, "Agent": "SAC"}, {"env_step": 785000, "rew": 43.2259, "rew_std": 1.0272, "Agent": "SAC"}, {"env_step": 790000, "rew": 43.4329, "rew_std": 0.8463, "Agent": "SAC"}, {"env_step": 795000, "rew": 43.2807, "rew_std": 0.6291, "Agent": "SAC"}, {"env_step": 800000, "rew": 43.2571, "rew_std": 1.088, "Agent": "SAC"}, {"env_step": 805000, "rew": 43.4552, "rew_std": 1.1519, "Agent": "SAC"}, {"env_step": 810000, "rew": 43.4323, "rew_std": 0.9875, "Agent": "SAC"}, {"env_step": 815000, "rew": 43.474, "rew_std": 0.9647, "Agent": "SAC"}, {"env_step": 820000, "rew": 43.0198, "rew_std": 1.1391, "Agent": "SAC"}, {"env_step": 825000, "rew": 42.789, "rew_std": 1.2367, "Agent": "SAC"}, {"env_step": 830000, "rew": 42.3454, "rew_std": 1.3127, "Agent": "SAC"}, {"env_step": 835000, "rew": 43.4136, "rew_std": 1.0791, "Agent": "SAC"}, {"env_step": 840000, "rew": 43.3779, "rew_std": 0.7225, "Agent": "SAC"}, {"env_step": 845000, "rew": 43.3594, "rew_std": 0.7372, "Agent": "SAC"}, {"env_step": 850000, "rew": 43.1062, "rew_std": 1.266, "Agent": "SAC"}, {"env_step": 855000, "rew": 42.6848, "rew_std": 0.9429, "Agent": "SAC"}, {"env_step": 860000, "rew": 43.5006, "rew_std": 0.8984, "Agent": "SAC"}, {"env_step": 865000, "rew": 43.2251, "rew_std": 1.3076, "Agent": "SAC"}, {"env_step": 870000, "rew": 43.7185, "rew_std": 0.9205, "Agent": "SAC"}, {"env_step": 875000, "rew": 43.6287, "rew_std": 1.03, "Agent": "SAC"}, {"env_step": 880000, "rew": 42.8404, "rew_std": 1.0181, "Agent": "SAC"}, {"env_step": 885000, "rew": 42.6213, "rew_std": 1.5867, "Agent": "SAC"}, {"env_step": 890000, "rew": 43.4013, "rew_std": 2.1414, "Agent": "SAC"}, {"env_step": 895000, "rew": 43.3951, "rew_std": 1.0499, "Agent": "SAC"}, {"env_step": 900000, "rew": 43.3527, "rew_std": 1.0789, "Agent": "SAC"}, {"env_step": 905000, "rew": 42.8644, "rew_std": 1.422, "Agent": "SAC"}, {"env_step": 910000, "rew": 42.9771, "rew_std": 1.0238, "Agent": "SAC"}, {"env_step": 915000, "rew": 43.5087, "rew_std": 1.0089, "Agent": "SAC"}, {"env_step": 920000, "rew": 43.373, "rew_std": 0.9359, "Agent": "SAC"}, {"env_step": 925000, "rew": 43.8177, "rew_std": 0.9008, "Agent": "SAC"}, {"env_step": 930000, "rew": 42.797, "rew_std": 0.8653, "Agent": "SAC"}, {"env_step": 935000, "rew": 43.276, "rew_std": 0.8909, "Agent": "SAC"}, {"env_step": 940000, "rew": 43.6976, "rew_std": 1.3921, "Agent": "SAC"}, {"env_step": 945000, "rew": 44.0239, "rew_std": 1.5283, "Agent": "SAC"}, {"env_step": 950000, "rew": 43.6672, "rew_std": 0.9988, "Agent": "SAC"}, {"env_step": 955000, "rew": 43.871, "rew_std": 0.9807, "Agent": "SAC"}, {"env_step": 960000, "rew": 44.3549, "rew_std": 0.4953, "Agent": "SAC"}, {"env_step": 965000, "rew": 43.8333, "rew_std": 1.216, "Agent": "SAC"}, {"env_step": 970000, "rew": 42.7589, "rew_std": 1.282, "Agent": "SAC"}, {"env_step": 975000, "rew": 43.4426, "rew_std": 1.0953, "Agent": "SAC"}, {"env_step": 980000, "rew": 43.913, "rew_std": 1.4897, "Agent": "SAC"}, {"env_step": 985000, "rew": 43.1539, "rew_std": 1.1106, "Agent": "SAC"}, {"env_step": 990000, "rew": 43.6382, "rew_std": 0.874, "Agent": "SAC"}, {"env_step": 995000, "rew": 43.3717, "rew_std": 2.2402, "Agent": "SAC"}, {"env_step": 1000000, "rew": 43.5731, "rew_std": 0.9632, "Agent": "SAC"}, {"env_step": 1005000, "rew": 44.2985, "rew_std": 1.4402, "Agent": "SAC"}, {"env_step": 1010000, "rew": 43.1896, "rew_std": 1.0631, "Agent": "SAC"}, {"env_step": 1015000, "rew": 43.7741, "rew_std": 1.1827, "Agent": "SAC"}, {"env_step": 1020000, "rew": 43.3785, "rew_std": 1.3209, "Agent": "SAC"}, {"env_step": 1025000, "rew": 43.4507, "rew_std": 0.8396, "Agent": "SAC"}, {"env_step": 1030000, "rew": 43.0527, "rew_std": 0.7842, "Agent": "SAC"}, {"env_step": 1035000, "rew": 43.5944, "rew_std": 0.8916, "Agent": "SAC"}, {"env_step": 1040000, "rew": 43.0462, "rew_std": 0.8515, "Agent": "SAC"}, {"env_step": 1045000, "rew": 43.3839, "rew_std": 0.5822, "Agent": "SAC"}, {"env_step": 1050000, "rew": 43.4203, "rew_std": 1.0907, "Agent": "SAC"}, {"env_step": 1055000, "rew": 43.3705, "rew_std": 0.9512, "Agent": "SAC"}, {"env_step": 1060000, "rew": 43.7978, "rew_std": 0.704, "Agent": "SAC"}, {"env_step": 1065000, "rew": 43.7268, "rew_std": 0.9554, "Agent": "SAC"}, {"env_step": 1070000, "rew": 43.5983, "rew_std": 0.7237, "Agent": "SAC"}, {"env_step": 1075000, "rew": 43.5771, "rew_std": 1.0144, "Agent": "SAC"}, {"env_step": 1080000, "rew": 44.0056, "rew_std": 1.0758, "Agent": "SAC"}, {"env_step": 1085000, "rew": 44.1718, "rew_std": 0.9893, "Agent": "SAC"}, {"env_step": 1090000, "rew": 43.1443, "rew_std": 0.9257, "Agent": "SAC"}, {"env_step": 1095000, "rew": 43.5743, "rew_std": 1.2003, "Agent": "SAC"}, {"env_step": 1100000, "rew": 43.8866, "rew_std": 0.8057, "Agent": "SAC"}, {"env_step": 1105000, "rew": 44.0473, "rew_std": 0.876, "Agent": "SAC"}, {"env_step": 1110000, "rew": 43.4748, "rew_std": 0.9671, "Agent": "SAC"}, {"env_step": 1115000, "rew": 43.2211, "rew_std": 1.2039, "Agent": "SAC"}, {"env_step": 1120000, "rew": 44.6832, "rew_std": 0.8636, "Agent": "SAC"}, {"env_step": 1125000, "rew": 43.3888, "rew_std": 0.914, "Agent": "SAC"}, {"env_step": 1130000, "rew": 43.9367, "rew_std": 0.9688, "Agent": "SAC"}, {"env_step": 1135000, "rew": 44.0778, "rew_std": 0.8398, "Agent": "SAC"}, {"env_step": 1140000, "rew": 43.6552, "rew_std": 0.8414, "Agent": "SAC"}, {"env_step": 1145000, "rew": 43.8613, "rew_std": 0.7711, "Agent": "SAC"}, {"env_step": 1150000, "rew": 43.7305, "rew_std": 0.9517, "Agent": "SAC"}, {"env_step": 1155000, "rew": 44.4041, "rew_std": 0.7634, "Agent": "SAC"}, {"env_step": 1160000, "rew": 44.1468, "rew_std": 0.5902, "Agent": "SAC"}, {"env_step": 1165000, "rew": 43.6025, "rew_std": 0.4915, "Agent": "SAC"}, {"env_step": 1170000, "rew": 43.6958, "rew_std": 1.1437, "Agent": "SAC"}, {"env_step": 1175000, "rew": 43.7657, "rew_std": 1.3804, "Agent": "SAC"}, {"env_step": 1180000, "rew": 43.9155, "rew_std": 0.9774, "Agent": "SAC"}, {"env_step": 1185000, "rew": 43.7091, "rew_std": 0.7425, "Agent": "SAC"}, {"env_step": 1190000, "rew": 43.9555, "rew_std": 0.5798, "Agent": "SAC"}, {"env_step": 1195000, "rew": 43.2931, "rew_std": 0.8764, "Agent": "SAC"}, {"env_step": 1200000, "rew": 43.1039, "rew_std": 2.1058, "Agent": "SAC"}, {"env_step": 1205000, "rew": 43.7402, "rew_std": 0.7227, "Agent": "SAC"}, {"env_step": 1210000, "rew": 43.0009, "rew_std": 0.7026, "Agent": "SAC"}, {"env_step": 1215000, "rew": 43.0386, "rew_std": 1.1108, "Agent": "SAC"}, {"env_step": 1220000, "rew": 43.8467, "rew_std": 1.314, "Agent": "SAC"}, {"env_step": 1225000, "rew": 44.4796, "rew_std": 1.3949, "Agent": "SAC"}, {"env_step": 1230000, "rew": 43.759, "rew_std": 1.2652, "Agent": "SAC"}, {"env_step": 1235000, "rew": 44.294, "rew_std": 0.8111, "Agent": "SAC"}, {"env_step": 1240000, "rew": 42.8652, "rew_std": 2.2385, "Agent": "SAC"}, {"env_step": 1245000, "rew": 43.5858, "rew_std": 0.4343, "Agent": "SAC"}, {"env_step": 1250000, "rew": 43.7967, "rew_std": 1.4092, "Agent": "SAC"}, {"env_step": 0, "rew": -7.7439, "rew_std": 16.8878, "Agent": "DDPG"}, {"env_step": 5000, "rew": 27.111, "rew_std": 10.2362, "Agent": "DDPG"}, {"env_step": 10000, "rew": 28.6777, "rew_std": 26.9079, "Agent": "DDPG"}, {"env_step": 15000, "rew": 29.5172, "rew_std": 20.6009, "Agent": "DDPG"}, {"env_step": 20000, "rew": 31.1824, "rew_std": 12.3387, "Agent": "DDPG"}, {"env_step": 25000, "rew": 35.9658, "rew_std": 11.5662, "Agent": "DDPG"}, {"env_step": 30000, "rew": 32.4928, "rew_std": 11.4795, "Agent": "DDPG"}, {"env_step": 35000, "rew": 36.4445, "rew_std": 13.4968, "Agent": "DDPG"}, {"env_step": 40000, "rew": 29.8878, "rew_std": 17.7985, "Agent": "DDPG"}, {"env_step": 45000, "rew": 41.0883, "rew_std": 10.6527, "Agent": "DDPG"}, {"env_step": 50000, "rew": 39.2044, "rew_std": 5.6556, "Agent": "DDPG"}, {"env_step": 55000, "rew": 46.6439, "rew_std": 8.09, "Agent": "DDPG"}, {"env_step": 60000, "rew": 41.5579, "rew_std": 10.6506, "Agent": "DDPG"}, {"env_step": 65000, "rew": 44.8723, "rew_std": 10.5025, "Agent": "DDPG"}, {"env_step": 70000, "rew": 45.1928, "rew_std": 10.4595, "Agent": "DDPG"}, {"env_step": 75000, "rew": 41.8443, "rew_std": 10.1912, "Agent": "DDPG"}, {"env_step": 80000, "rew": 49.2066, "rew_std": 11.219, "Agent": "DDPG"}, {"env_step": 85000, "rew": 47.8705, "rew_std": 6.2717, "Agent": "DDPG"}, {"env_step": 90000, "rew": 51.3617, "rew_std": 6.4001, "Agent": "DDPG"}, {"env_step": 95000, "rew": 53.8505, "rew_std": 19.8274, "Agent": "DDPG"}, {"env_step": 100000, "rew": 54.9865, "rew_std": 14.7739, "Agent": "DDPG"}, {"env_step": 105000, "rew": 51.164, "rew_std": 5.3692, "Agent": "DDPG"}, {"env_step": 110000, "rew": 53.5985, "rew_std": 16.8755, "Agent": "DDPG"}, {"env_step": 115000, "rew": 53.6093, "rew_std": 16.448, "Agent": "DDPG"}, {"env_step": 120000, "rew": 54.4763, "rew_std": 6.8215, "Agent": "DDPG"}, {"env_step": 125000, "rew": 59.7466, "rew_std": 8.9492, "Agent": "DDPG"}, {"env_step": 130000, "rew": 58.1804, "rew_std": 10.4326, "Agent": "DDPG"}, {"env_step": 135000, "rew": 60.8608, "rew_std": 17.5299, "Agent": "DDPG"}, {"env_step": 140000, "rew": 61.3081, "rew_std": 19.6055, "Agent": "DDPG"}, {"env_step": 145000, "rew": 71.2575, "rew_std": 28.7301, "Agent": "DDPG"}, {"env_step": 150000, "rew": 64.1452, "rew_std": 22.4875, "Agent": "DDPG"}, {"env_step": 155000, "rew": 72.976, "rew_std": 23.0466, "Agent": "DDPG"}, {"env_step": 160000, "rew": 77.025, "rew_std": 28.1836, "Agent": "DDPG"}, {"env_step": 165000, "rew": 79.2977, "rew_std": 27.8582, "Agent": "DDPG"}, {"env_step": 170000, "rew": 77.7769, "rew_std": 27.7826, "Agent": "DDPG"}, {"env_step": 175000, "rew": 79.8337, "rew_std": 32.1006, "Agent": "DDPG"}, {"env_step": 180000, "rew": 86.3861, "rew_std": 29.9729, "Agent": "DDPG"}, {"env_step": 185000, "rew": 83.9423, "rew_std": 32.7325, "Agent": "DDPG"}, {"env_step": 190000, "rew": 87.4571, "rew_std": 29.596, "Agent": "DDPG"}, {"env_step": 195000, "rew": 88.5118, "rew_std": 34.1182, "Agent": "DDPG"}, {"env_step": 200000, "rew": 96.4085, "rew_std": 30.9893, "Agent": "DDPG"}, {"env_step": 205000, "rew": 90.2024, "rew_std": 27.9598, "Agent": "DDPG"}, {"env_step": 210000, "rew": 100.2493, "rew_std": 27.3851, "Agent": "DDPG"}, {"env_step": 215000, "rew": 100.3753, "rew_std": 29.3593, "Agent": "DDPG"}, {"env_step": 220000, "rew": 104.0079, "rew_std": 25.5209, "Agent": "DDPG"}, {"env_step": 225000, "rew": 104.117, "rew_std": 24.1255, "Agent": "DDPG"}, {"env_step": 230000, "rew": 101.1801, "rew_std": 27.2732, "Agent": "DDPG"}, {"env_step": 235000, "rew": 113.4582, "rew_std": 23.629, "Agent": "DDPG"}, {"env_step": 240000, "rew": 113.4486, "rew_std": 23.8049, "Agent": "DDPG"}, {"env_step": 245000, "rew": 111.7019, "rew_std": 25.4914, "Agent": "DDPG"}, {"env_step": 250000, "rew": 115.0514, "rew_std": 24.8545, "Agent": "DDPG"}, {"env_step": 255000, "rew": 115.979, "rew_std": 24.0976, "Agent": "DDPG"}, {"env_step": 260000, "rew": 122.0173, "rew_std": 24.6718, "Agent": "DDPG"}, {"env_step": 265000, "rew": 121.0905, "rew_std": 23.2614, "Agent": "DDPG"}, {"env_step": 270000, "rew": 118.8829, "rew_std": 28.7329, "Agent": "DDPG"}, {"env_step": 275000, "rew": 121.1204, "rew_std": 22.4617, "Agent": "DDPG"}, {"env_step": 280000, "rew": 120.106, "rew_std": 22.087, "Agent": "DDPG"}, {"env_step": 285000, "rew": 122.4843, "rew_std": 25.8522, "Agent": "DDPG"}, {"env_step": 290000, "rew": 126.2871, "rew_std": 19.0177, "Agent": "DDPG"}, {"env_step": 295000, "rew": 121.7824, "rew_std": 19.3339, "Agent": "DDPG"}, {"env_step": 300000, "rew": 124.2313, "rew_std": 22.2559, "Agent": "DDPG"}, {"env_step": 305000, "rew": 127.2795, "rew_std": 18.6246, "Agent": "DDPG"}, {"env_step": 310000, "rew": 126.778, "rew_std": 19.8358, "Agent": "DDPG"}, {"env_step": 315000, "rew": 128.4156, "rew_std": 21.5798, "Agent": "DDPG"}, {"env_step": 320000, "rew": 126.0861, "rew_std": 20.0165, "Agent": "DDPG"}, {"env_step": 325000, "rew": 129.3534, "rew_std": 19.0216, "Agent": "DDPG"}, {"env_step": 330000, "rew": 130.358, "rew_std": 17.8578, "Agent": "DDPG"}, {"env_step": 335000, "rew": 132.1706, "rew_std": 13.9828, "Agent": "DDPG"}, {"env_step": 340000, "rew": 129.9149, "rew_std": 12.1114, "Agent": "DDPG"}, {"env_step": 345000, "rew": 131.798, "rew_std": 15.8348, "Agent": "DDPG"}, {"env_step": 350000, "rew": 131.8573, "rew_std": 13.4496, "Agent": "DDPG"}, {"env_step": 355000, "rew": 130.0461, "rew_std": 11.5573, "Agent": "DDPG"}, {"env_step": 360000, "rew": 129.0096, "rew_std": 12.2683, "Agent": "DDPG"}, {"env_step": 365000, "rew": 129.1626, "rew_std": 9.9854, "Agent": "DDPG"}, {"env_step": 370000, "rew": 128.7662, "rew_std": 9.881, "Agent": "DDPG"}, {"env_step": 375000, "rew": 130.6815, "rew_std": 9.6643, "Agent": "DDPG"}, {"env_step": 380000, "rew": 132.8223, "rew_std": 10.335, "Agent": "DDPG"}, {"env_step": 385000, "rew": 133.3924, "rew_std": 13.4702, "Agent": "DDPG"}, {"env_step": 390000, "rew": 133.1555, "rew_std": 11.541, "Agent": "DDPG"}, {"env_step": 395000, "rew": 131.1679, "rew_std": 8.1387, "Agent": "DDPG"}, {"env_step": 400000, "rew": 129.9687, "rew_std": 15.068, "Agent": "DDPG"}, {"env_step": 405000, "rew": 134.1776, "rew_std": 10.885, "Agent": "DDPG"}, {"env_step": 410000, "rew": 136.1757, "rew_std": 9.1146, "Agent": "DDPG"}, {"env_step": 415000, "rew": 133.9262, "rew_std": 12.703, "Agent": "DDPG"}, {"env_step": 420000, "rew": 136.7213, "rew_std": 11.2332, "Agent": "DDPG"}, {"env_step": 425000, "rew": 135.8019, "rew_std": 12.0867, "Agent": "DDPG"}, {"env_step": 430000, "rew": 137.1716, "rew_std": 11.1026, "Agent": "DDPG"}, {"env_step": 435000, "rew": 137.9368, "rew_std": 10.9157, "Agent": "DDPG"}, {"env_step": 440000, "rew": 135.5737, "rew_std": 12.0013, "Agent": "DDPG"}, {"env_step": 445000, "rew": 136.3091, "rew_std": 12.8528, "Agent": "DDPG"}, {"env_step": 450000, "rew": 132.9403, "rew_std": 13.7779, "Agent": "DDPG"}, {"env_step": 455000, "rew": 137.4564, "rew_std": 7.8761, "Agent": "DDPG"}, {"env_step": 460000, "rew": 138.432, "rew_std": 7.0544, "Agent": "DDPG"}, {"env_step": 465000, "rew": 133.4846, "rew_std": 8.8001, "Agent": "DDPG"}, {"env_step": 470000, "rew": 133.5229, "rew_std": 10.4753, "Agent": "DDPG"}, {"env_step": 475000, "rew": 135.3913, "rew_std": 9.919, "Agent": "DDPG"}, {"env_step": 480000, "rew": 132.1134, "rew_std": 26.522, "Agent": "DDPG"}, {"env_step": 485000, "rew": 138.0401, "rew_std": 13.214, "Agent": "DDPG"}, {"env_step": 490000, "rew": 137.6227, "rew_std": 9.5667, "Agent": "DDPG"}, {"env_step": 495000, "rew": 132.7349, "rew_std": 15.8578, "Agent": "DDPG"}, {"env_step": 500000, "rew": 136.57, "rew_std": 6.4353, "Agent": "DDPG"}, {"env_step": 505000, "rew": 139.3636, "rew_std": 9.0841, "Agent": "DDPG"}, {"env_step": 510000, "rew": 136.9163, "rew_std": 7.471, "Agent": "DDPG"}, {"env_step": 515000, "rew": 136.207, "rew_std": 9.6977, "Agent": "DDPG"}, {"env_step": 520000, "rew": 138.2055, "rew_std": 6.4213, "Agent": "DDPG"}, {"env_step": 525000, "rew": 136.4244, "rew_std": 5.5003, "Agent": "DDPG"}, {"env_step": 530000, "rew": 136.5625, "rew_std": 4.0473, "Agent": "DDPG"}, {"env_step": 535000, "rew": 138.2565, "rew_std": 5.4598, "Agent": "DDPG"}, {"env_step": 540000, "rew": 137.4138, "rew_std": 7.1241, "Agent": "DDPG"}, {"env_step": 545000, "rew": 140.4092, "rew_std": 10.8827, "Agent": "DDPG"}, {"env_step": 550000, "rew": 138.3283, "rew_std": 8.6329, "Agent": "DDPG"}, {"env_step": 555000, "rew": 136.1647, "rew_std": 7.8454, "Agent": "DDPG"}, {"env_step": 560000, "rew": 140.6664, "rew_std": 6.8785, "Agent": "DDPG"}, {"env_step": 565000, "rew": 139.045, "rew_std": 9.4246, "Agent": "DDPG"}, {"env_step": 570000, "rew": 135.7369, "rew_std": 9.8926, "Agent": "DDPG"}, {"env_step": 575000, "rew": 136.6039, "rew_std": 6.9162, "Agent": "DDPG"}, {"env_step": 580000, "rew": 138.3082, "rew_std": 5.7867, "Agent": "DDPG"}, {"env_step": 585000, "rew": 139.113, "rew_std": 7.4853, "Agent": "DDPG"}, {"env_step": 590000, "rew": 139.5336, "rew_std": 8.0436, "Agent": "DDPG"}, {"env_step": 595000, "rew": 135.7219, "rew_std": 9.0493, "Agent": "DDPG"}, {"env_step": 600000, "rew": 139.0518, "rew_std": 5.966, "Agent": "DDPG"}, {"env_step": 605000, "rew": 134.7683, "rew_std": 10.3015, "Agent": "DDPG"}, {"env_step": 610000, "rew": 138.8106, "rew_std": 8.7777, "Agent": "DDPG"}, {"env_step": 615000, "rew": 137.989, "rew_std": 8.3029, "Agent": "DDPG"}, {"env_step": 620000, "rew": 135.4547, "rew_std": 11.4179, "Agent": "DDPG"}, {"env_step": 625000, "rew": 138.5979, "rew_std": 13.4474, "Agent": "DDPG"}, {"env_step": 630000, "rew": 137.1638, "rew_std": 4.0057, "Agent": "DDPG"}, {"env_step": 635000, "rew": 138.6027, "rew_std": 6.9118, "Agent": "DDPG"}, {"env_step": 640000, "rew": 138.4561, "rew_std": 5.3458, "Agent": "DDPG"}, {"env_step": 645000, "rew": 137.6744, "rew_std": 9.7438, "Agent": "DDPG"}, {"env_step": 650000, "rew": 140.8803, "rew_std": 10.5245, "Agent": "DDPG"}, {"env_step": 655000, "rew": 137.6448, "rew_std": 10.9617, "Agent": "DDPG"}, {"env_step": 660000, "rew": 140.081, "rew_std": 6.8093, "Agent": "DDPG"}, {"env_step": 665000, "rew": 140.6238, "rew_std": 6.7335, "Agent": "DDPG"}, {"env_step": 670000, "rew": 137.5476, "rew_std": 5.7097, "Agent": "DDPG"}, {"env_step": 675000, "rew": 140.0157, "rew_std": 3.6635, "Agent": "DDPG"}, {"env_step": 680000, "rew": 139.8821, "rew_std": 6.7079, "Agent": "DDPG"}, {"env_step": 685000, "rew": 139.6744, "rew_std": 4.1447, "Agent": "DDPG"}, {"env_step": 690000, "rew": 140.5739, "rew_std": 3.8759, "Agent": "DDPG"}, {"env_step": 695000, "rew": 140.9771, "rew_std": 5.3847, "Agent": "DDPG"}, {"env_step": 700000, "rew": 141.7002, "rew_std": 4.175, "Agent": "DDPG"}, {"env_step": 705000, "rew": 140.0991, "rew_std": 6.4371, "Agent": "DDPG"}, {"env_step": 710000, "rew": 138.9222, "rew_std": 4.4813, "Agent": "DDPG"}, {"env_step": 715000, "rew": 140.0551, "rew_std": 9.8149, "Agent": "DDPG"}, {"env_step": 720000, "rew": 140.2259, "rew_std": 4.8199, "Agent": "DDPG"}, {"env_step": 725000, "rew": 138.2153, "rew_std": 8.7884, "Agent": "DDPG"}, {"env_step": 730000, "rew": 139.7536, "rew_std": 6.6101, "Agent": "DDPG"}, {"env_step": 735000, "rew": 140.7631, "rew_std": 4.6529, "Agent": "DDPG"}, {"env_step": 740000, "rew": 138.884, "rew_std": 5.6056, "Agent": "DDPG"}, {"env_step": 745000, "rew": 142.3084, "rew_std": 7.3071, "Agent": "DDPG"}, {"env_step": 750000, "rew": 143.9028, "rew_std": 5.4536, "Agent": "DDPG"}, {"env_step": 755000, "rew": 143.2372, "rew_std": 7.275, "Agent": "DDPG"}, {"env_step": 760000, "rew": 140.6356, "rew_std": 4.3539, "Agent": "DDPG"}, {"env_step": 765000, "rew": 137.9779, "rew_std": 6.4228, "Agent": "DDPG"}, {"env_step": 770000, "rew": 139.6024, "rew_std": 4.6246, "Agent": "DDPG"}, {"env_step": 775000, "rew": 139.0619, "rew_std": 7.1484, "Agent": "DDPG"}, {"env_step": 780000, "rew": 138.1545, "rew_std": 11.656, "Agent": "DDPG"}, {"env_step": 785000, "rew": 138.0749, "rew_std": 3.3782, "Agent": "DDPG"}, {"env_step": 790000, "rew": 140.7768, "rew_std": 8.0989, "Agent": "DDPG"}, {"env_step": 795000, "rew": 138.3206, "rew_std": 7.3118, "Agent": "DDPG"}, {"env_step": 800000, "rew": 140.6746, "rew_std": 4.9365, "Agent": "DDPG"}, {"env_step": 805000, "rew": 142.133, "rew_std": 3.0798, "Agent": "DDPG"}, {"env_step": 810000, "rew": 141.6982, "rew_std": 5.9378, "Agent": "DDPG"}, {"env_step": 815000, "rew": 139.0957, "rew_std": 5.2462, "Agent": "DDPG"}, {"env_step": 820000, "rew": 141.4906, "rew_std": 5.9738, "Agent": "DDPG"}, {"env_step": 825000, "rew": 139.0263, "rew_std": 8.3833, "Agent": "DDPG"}, {"env_step": 830000, "rew": 142.5686, "rew_std": 7.9465, "Agent": "DDPG"}, {"env_step": 835000, "rew": 139.6923, "rew_std": 4.4107, "Agent": "DDPG"}, {"env_step": 840000, "rew": 141.8938, "rew_std": 4.7214, "Agent": "DDPG"}, {"env_step": 845000, "rew": 140.2036, "rew_std": 3.3937, "Agent": "DDPG"}, {"env_step": 850000, "rew": 138.7109, "rew_std": 5.7588, "Agent": "DDPG"}, {"env_step": 855000, "rew": 140.7613, "rew_std": 7.2326, "Agent": "DDPG"}, {"env_step": 860000, "rew": 141.6586, "rew_std": 7.8614, "Agent": "DDPG"}, {"env_step": 865000, "rew": 140.926, "rew_std": 4.7326, "Agent": "DDPG"}, {"env_step": 870000, "rew": 139.1153, "rew_std": 6.172, "Agent": "DDPG"}, {"env_step": 875000, "rew": 142.6159, "rew_std": 3.4461, "Agent": "DDPG"}, {"env_step": 880000, "rew": 142.8545, "rew_std": 6.1448, "Agent": "DDPG"}, {"env_step": 885000, "rew": 141.4641, "rew_std": 6.418, "Agent": "DDPG"}, {"env_step": 890000, "rew": 142.9374, "rew_std": 6.063, "Agent": "DDPG"}, {"env_step": 895000, "rew": 142.308, "rew_std": 6.3968, "Agent": "DDPG"}, {"env_step": 900000, "rew": 141.8166, "rew_std": 4.2354, "Agent": "DDPG"}, {"env_step": 905000, "rew": 141.3372, "rew_std": 3.6162, "Agent": "DDPG"}, {"env_step": 910000, "rew": 142.54, "rew_std": 7.2792, "Agent": "DDPG"}, {"env_step": 915000, "rew": 142.4178, "rew_std": 4.8727, "Agent": "DDPG"}, {"env_step": 920000, "rew": 139.7616, "rew_std": 9.0963, "Agent": "DDPG"}, {"env_step": 925000, "rew": 140.8662, "rew_std": 7.1925, "Agent": "DDPG"}, {"env_step": 930000, "rew": 142.59, "rew_std": 7.5659, "Agent": "DDPG"}, {"env_step": 935000, "rew": 141.1232, "rew_std": 5.4951, "Agent": "DDPG"}, {"env_step": 940000, "rew": 140.9761, "rew_std": 4.436, "Agent": "DDPG"}, {"env_step": 945000, "rew": 142.694, "rew_std": 6.0702, "Agent": "DDPG"}, {"env_step": 950000, "rew": 143.414, "rew_std": 7.787, "Agent": "DDPG"}, {"env_step": 955000, "rew": 141.0688, "rew_std": 5.583, "Agent": "DDPG"}, {"env_step": 960000, "rew": 141.8118, "rew_std": 8.1901, "Agent": "DDPG"}, {"env_step": 965000, "rew": 141.6261, "rew_std": 6.3236, "Agent": "DDPG"}, {"env_step": 970000, "rew": 142.2169, "rew_std": 7.4153, "Agent": "DDPG"}, {"env_step": 975000, "rew": 139.8413, "rew_std": 5.3582, "Agent": "DDPG"}, {"env_step": 980000, "rew": 141.311, "rew_std": 7.0072, "Agent": "DDPG"}, {"env_step": 985000, "rew": 140.5871, "rew_std": 5.2723, "Agent": "DDPG"}, {"env_step": 990000, "rew": 144.0535, "rew_std": 6.4924, "Agent": "DDPG"}, {"env_step": 995000, "rew": 141.8434, "rew_std": 4.2558, "Agent": "DDPG"}, {"env_step": 1000000, "rew": 142.2982, "rew_std": 7.6682, "Agent": "DDPG"}, {"env_step": 1005000, "rew": 139.1055, "rew_std": 9.2758, "Agent": "DDPG"}, {"env_step": 1010000, "rew": 141.0416, "rew_std": 5.0772, "Agent": "DDPG"}, {"env_step": 1015000, "rew": 141.6343, "rew_std": 6.0345, "Agent": "DDPG"}, {"env_step": 1020000, "rew": 141.7569, "rew_std": 4.6563, "Agent": "DDPG"}, {"env_step": 1025000, "rew": 141.4089, "rew_std": 4.7605, "Agent": "DDPG"}, {"env_step": 1030000, "rew": 141.2138, "rew_std": 5.7038, "Agent": "DDPG"}, {"env_step": 1035000, "rew": 144.4552, "rew_std": 6.0544, "Agent": "DDPG"}, {"env_step": 1040000, "rew": 140.9662, "rew_std": 3.1099, "Agent": "DDPG"}, {"env_step": 1045000, "rew": 141.1972, "rew_std": 4.5801, "Agent": "DDPG"}, {"env_step": 1050000, "rew": 140.5965, "rew_std": 7.0805, "Agent": "DDPG"}, {"env_step": 1055000, "rew": 140.1733, "rew_std": 4.7662, "Agent": "DDPG"}, {"env_step": 1060000, "rew": 140.4672, "rew_std": 5.3948, "Agent": "DDPG"}, {"env_step": 1065000, "rew": 143.2305, "rew_std": 3.5909, "Agent": "DDPG"}, {"env_step": 1070000, "rew": 143.6553, "rew_std": 5.7578, "Agent": "DDPG"}, {"env_step": 1075000, "rew": 142.2234, "rew_std": 7.5004, "Agent": "DDPG"}, {"env_step": 1080000, "rew": 139.3473, "rew_std": 10.9533, "Agent": "DDPG"}, {"env_step": 1085000, "rew": 140.791, "rew_std": 8.3328, "Agent": "DDPG"}, {"env_step": 1090000, "rew": 141.3331, "rew_std": 9.4282, "Agent": "DDPG"}, {"env_step": 1095000, "rew": 143.6647, "rew_std": 6.6349, "Agent": "DDPG"}, {"env_step": 1100000, "rew": 142.53, "rew_std": 5.9298, "Agent": "DDPG"}, {"env_step": 1105000, "rew": 141.2602, "rew_std": 6.2225, "Agent": "DDPG"}, {"env_step": 1110000, "rew": 140.7595, "rew_std": 4.7542, "Agent": "DDPG"}, {"env_step": 1115000, "rew": 140.6618, "rew_std": 5.5336, "Agent": "DDPG"}, {"env_step": 1120000, "rew": 142.304, "rew_std": 8.0534, "Agent": "DDPG"}, {"env_step": 1125000, "rew": 144.1979, "rew_std": 3.9518, "Agent": "DDPG"}, {"env_step": 1130000, "rew": 141.9399, "rew_std": 5.1798, "Agent": "DDPG"}, {"env_step": 1135000, "rew": 141.6419, "rew_std": 5.5409, "Agent": "DDPG"}, {"env_step": 1140000, "rew": 141.4458, "rew_std": 5.9887, "Agent": "DDPG"}, {"env_step": 1145000, "rew": 140.9982, "rew_std": 5.9734, "Agent": "DDPG"}, {"env_step": 1150000, "rew": 142.6737, "rew_std": 9.5289, "Agent": "DDPG"}, {"env_step": 1155000, "rew": 141.9587, "rew_std": 9.3877, "Agent": "DDPG"}, {"env_step": 1160000, "rew": 137.8503, "rew_std": 14.5437, "Agent": "DDPG"}, {"env_step": 1165000, "rew": 139.3552, "rew_std": 21.4731, "Agent": "DDPG"}, {"env_step": 1170000, "rew": 141.687, "rew_std": 7.7627, "Agent": "DDPG"}, {"env_step": 1175000, "rew": 142.3425, "rew_std": 7.2573, "Agent": "DDPG"}, {"env_step": 1180000, "rew": 138.1963, "rew_std": 16.6084, "Agent": "DDPG"}, {"env_step": 1185000, "rew": 141.6446, "rew_std": 6.3986, "Agent": "DDPG"}, {"env_step": 1190000, "rew": 143.3347, "rew_std": 9.8443, "Agent": "DDPG"}, {"env_step": 1195000, "rew": 140.4516, "rew_std": 6.6021, "Agent": "DDPG"}, {"env_step": 1200000, "rew": 140.1266, "rew_std": 5.6212, "Agent": "DDPG"}, {"env_step": 1205000, "rew": 143.1696, "rew_std": 5.1804, "Agent": "DDPG"}, {"env_step": 1210000, "rew": 143.2144, "rew_std": 4.4858, "Agent": "DDPG"}, {"env_step": 1215000, "rew": 138.569, "rew_std": 12.7858, "Agent": "DDPG"}, {"env_step": 1220000, "rew": 141.7806, "rew_std": 5.4688, "Agent": "DDPG"}, {"env_step": 1225000, "rew": 145.6152, "rew_std": 6.855, "Agent": "DDPG"}, {"env_step": 1230000, "rew": 137.5698, "rew_std": 21.3122, "Agent": "DDPG"}, {"env_step": 1235000, "rew": 143.4506, "rew_std": 7.6401, "Agent": "DDPG"}, {"env_step": 1240000, "rew": 144.9408, "rew_std": 7.5304, "Agent": "DDPG"}, {"env_step": 1245000, "rew": 135.8165, "rew_std": 15.8613, "Agent": "DDPG"}, {"env_step": 1250000, "rew": 140.8486, "rew_std": 11.2777, "Agent": "DDPG"}, {"env_step": 0, "rew": -7.7439, "rew_std": 16.8878, "Agent": "TD3"}, {"env_step": 5000, "rew": 24.6469, "rew_std": 4.5382, "Agent": "TD3"}, {"env_step": 10000, "rew": 25.0534, "rew_std": 10.6669, "Agent": "TD3"}, {"env_step": 15000, "rew": 29.6534, "rew_std": 11.6169, "Agent": "TD3"}, {"env_step": 20000, "rew": 38.585, "rew_std": 9.998, "Agent": "TD3"}, {"env_step": 25000, "rew": 44.6048, "rew_std": 11.0176, "Agent": "TD3"}, {"env_step": 30000, "rew": 26.28, "rew_std": 26.582, "Agent": "TD3"}, {"env_step": 35000, "rew": 34.9212, "rew_std": 8.0993, "Agent": "TD3"}, {"env_step": 40000, "rew": 33.5783, "rew_std": 11.7866, "Agent": "TD3"}, {"env_step": 45000, "rew": 33.854, "rew_std": 8.7302, "Agent": "TD3"}, {"env_step": 50000, "rew": 39.2372, "rew_std": 13.5948, "Agent": "TD3"}, {"env_step": 55000, "rew": 37.533, "rew_std": 6.0203, "Agent": "TD3"}, {"env_step": 60000, "rew": 36.653, "rew_std": 6.8221, "Agent": "TD3"}, {"env_step": 65000, "rew": 41.4642, "rew_std": 8.1535, "Agent": "TD3"}, {"env_step": 70000, "rew": 40.2504, "rew_std": 8.7733, "Agent": "TD3"}, {"env_step": 75000, "rew": 40.4649, "rew_std": 9.2816, "Agent": "TD3"}, {"env_step": 80000, "rew": 38.4952, "rew_std": 16.6833, "Agent": "TD3"}, {"env_step": 85000, "rew": 41.4131, "rew_std": 9.4847, "Agent": "TD3"}, {"env_step": 90000, "rew": 40.3345, "rew_std": 12.7181, "Agent": "TD3"}, {"env_step": 95000, "rew": 44.8763, "rew_std": 6.4609, "Agent": "TD3"}, {"env_step": 100000, "rew": 44.8324, "rew_std": 6.8199, "Agent": "TD3"}, {"env_step": 105000, "rew": 42.1155, "rew_std": 6.4451, "Agent": "TD3"}, {"env_step": 110000, "rew": 45.29, "rew_std": 3.558, "Agent": "TD3"}, {"env_step": 115000, "rew": 43.5655, "rew_std": 8.7519, "Agent": "TD3"}, {"env_step": 120000, "rew": 44.2056, "rew_std": 7.8904, "Agent": "TD3"}, {"env_step": 125000, "rew": 45.1126, "rew_std": 4.9606, "Agent": "TD3"}, {"env_step": 130000, "rew": 46.5372, "rew_std": 5.0502, "Agent": "TD3"}, {"env_step": 135000, "rew": 46.5835, "rew_std": 5.288, "Agent": "TD3"}, {"env_step": 140000, "rew": 48.8754, "rew_std": 4.7581, "Agent": "TD3"}, {"env_step": 145000, "rew": 46.3889, "rew_std": 3.3211, "Agent": "TD3"}, {"env_step": 150000, "rew": 45.6373, "rew_std": 3.9252, "Agent": "TD3"}, {"env_step": 155000, "rew": 46.1775, "rew_std": 3.5939, "Agent": "TD3"}, {"env_step": 160000, "rew": 47.9908, "rew_std": 4.1987, "Agent": "TD3"}, {"env_step": 165000, "rew": 49.3943, "rew_std": 5.5982, "Agent": "TD3"}, {"env_step": 170000, "rew": 48.5755, "rew_std": 4.1708, "Agent": "TD3"}, {"env_step": 175000, "rew": 48.1512, "rew_std": 4.4001, "Agent": "TD3"}, {"env_step": 180000, "rew": 48.9247, "rew_std": 3.3916, "Agent": "TD3"}, {"env_step": 185000, "rew": 49.942, "rew_std": 3.9773, "Agent": "TD3"}, {"env_step": 190000, "rew": 48.5643, "rew_std": 2.8522, "Agent": "TD3"}, {"env_step": 195000, "rew": 47.3657, "rew_std": 5.0301, "Agent": "TD3"}, {"env_step": 200000, "rew": 50.0434, "rew_std": 3.9202, "Agent": "TD3"}, {"env_step": 205000, "rew": 47.858, "rew_std": 3.4439, "Agent": "TD3"}, {"env_step": 210000, "rew": 50.2507, "rew_std": 4.5715, "Agent": "TD3"}, {"env_step": 215000, "rew": 51.8494, "rew_std": 7.9624, "Agent": "TD3"}, {"env_step": 220000, "rew": 49.5942, "rew_std": 6.0095, "Agent": "TD3"}, {"env_step": 225000, "rew": 49.2158, "rew_std": 4.0273, "Agent": "TD3"}, {"env_step": 230000, "rew": 50.2866, "rew_std": 7.6562, "Agent": "TD3"}, {"env_step": 235000, "rew": 51.362, "rew_std": 6.8194, "Agent": "TD3"}, {"env_step": 240000, "rew": 49.5634, "rew_std": 6.0966, "Agent": "TD3"}, {"env_step": 245000, "rew": 47.582, "rew_std": 11.166, "Agent": "TD3"}, {"env_step": 250000, "rew": 51.2911, "rew_std": 8.8841, "Agent": "TD3"}, {"env_step": 255000, "rew": 52.3947, "rew_std": 9.7762, "Agent": "TD3"}, {"env_step": 260000, "rew": 52.2859, "rew_std": 11.4657, "Agent": "TD3"}, {"env_step": 265000, "rew": 55.5904, "rew_std": 12.1624, "Agent": "TD3"}, {"env_step": 270000, "rew": 55.3218, "rew_std": 10.1369, "Agent": "TD3"}, {"env_step": 275000, "rew": 53.8998, "rew_std": 9.8109, "Agent": "TD3"}, {"env_step": 280000, "rew": 56.1154, "rew_std": 7.0174, "Agent": "TD3"}, {"env_step": 285000, "rew": 57.3024, "rew_std": 10.2827, "Agent": "TD3"}, {"env_step": 290000, "rew": 57.764, "rew_std": 11.1871, "Agent": "TD3"}, {"env_step": 295000, "rew": 59.0905, "rew_std": 17.8983, "Agent": "TD3"}, {"env_step": 300000, "rew": 59.292, "rew_std": 17.2291, "Agent": "TD3"}, {"env_step": 305000, "rew": 60.419, "rew_std": 17.6389, "Agent": "TD3"}, {"env_step": 310000, "rew": 61.5869, "rew_std": 20.8453, "Agent": "TD3"}, {"env_step": 315000, "rew": 58.665, "rew_std": 14.5205, "Agent": "TD3"}, {"env_step": 320000, "rew": 61.191, "rew_std": 17.4832, "Agent": "TD3"}, {"env_step": 325000, "rew": 62.7797, "rew_std": 18.1448, "Agent": "TD3"}, {"env_step": 330000, "rew": 62.6546, "rew_std": 19.4931, "Agent": "TD3"}, {"env_step": 335000, "rew": 65.461, "rew_std": 19.32, "Agent": "TD3"}, {"env_step": 340000, "rew": 65.9703, "rew_std": 22.4508, "Agent": "TD3"}, {"env_step": 345000, "rew": 64.5671, "rew_std": 21.895, "Agent": "TD3"}, {"env_step": 350000, "rew": 69.7419, "rew_std": 25.4201, "Agent": "TD3"}, {"env_step": 355000, "rew": 66.8439, "rew_std": 21.4126, "Agent": "TD3"}, {"env_step": 360000, "rew": 67.2735, "rew_std": 22.4134, "Agent": "TD3"}, {"env_step": 365000, "rew": 70.1827, "rew_std": 23.8667, "Agent": "TD3"}, {"env_step": 370000, "rew": 67.7494, "rew_std": 21.4642, "Agent": "TD3"}, {"env_step": 375000, "rew": 70.1232, "rew_std": 26.1163, "Agent": "TD3"}, {"env_step": 380000, "rew": 70.7586, "rew_std": 24.5781, "Agent": "TD3"}, {"env_step": 385000, "rew": 69.3465, "rew_std": 26.9114, "Agent": "TD3"}, {"env_step": 390000, "rew": 70.8394, "rew_std": 25.6505, "Agent": "TD3"}, {"env_step": 395000, "rew": 72.95, "rew_std": 28.7033, "Agent": "TD3"}, {"env_step": 400000, "rew": 74.0454, "rew_std": 30.5719, "Agent": "TD3"}, {"env_step": 405000, "rew": 72.0362, "rew_std": 29.6133, "Agent": "TD3"}, {"env_step": 410000, "rew": 70.9323, "rew_std": 26.7513, "Agent": "TD3"}, {"env_step": 415000, "rew": 73.8199, "rew_std": 30.3458, "Agent": "TD3"}, {"env_step": 420000, "rew": 72.9135, "rew_std": 31.2769, "Agent": "TD3"}, {"env_step": 425000, "rew": 70.1839, "rew_std": 24.8352, "Agent": "TD3"}, {"env_step": 430000, "rew": 72.7441, "rew_std": 25.5592, "Agent": "TD3"}, {"env_step": 435000, "rew": 77.4554, "rew_std": 29.2872, "Agent": "TD3"}, {"env_step": 440000, "rew": 77.9076, "rew_std": 30.3671, "Agent": "TD3"}, {"env_step": 445000, "rew": 77.3448, "rew_std": 31.0666, "Agent": "TD3"}, {"env_step": 450000, "rew": 76.5884, "rew_std": 31.2528, "Agent": "TD3"}, {"env_step": 455000, "rew": 78.3391, "rew_std": 33.6286, "Agent": "TD3"}, {"env_step": 460000, "rew": 77.0918, "rew_std": 33.4976, "Agent": "TD3"}, {"env_step": 465000, "rew": 76.8354, "rew_std": 32.2517, "Agent": "TD3"}, {"env_step": 470000, "rew": 76.2787, "rew_std": 30.1177, "Agent": "TD3"}, {"env_step": 475000, "rew": 78.2738, "rew_std": 32.6535, "Agent": "TD3"}, {"env_step": 480000, "rew": 76.5014, "rew_std": 28.3248, "Agent": "TD3"}, {"env_step": 485000, "rew": 76.188, "rew_std": 30.7491, "Agent": "TD3"}, {"env_step": 490000, "rew": 77.5848, "rew_std": 30.0843, "Agent": "TD3"}, {"env_step": 495000, "rew": 77.3351, "rew_std": 29.965, "Agent": "TD3"}, {"env_step": 500000, "rew": 79.3092, "rew_std": 30.0778, "Agent": "TD3"}, {"env_step": 505000, "rew": 74.8518, "rew_std": 30.9475, "Agent": "TD3"}, {"env_step": 510000, "rew": 80.657, "rew_std": 31.7103, "Agent": "TD3"}, {"env_step": 515000, "rew": 80.9163, "rew_std": 29.685, "Agent": "TD3"}, {"env_step": 520000, "rew": 80.2363, "rew_std": 29.1871, "Agent": "TD3"}, {"env_step": 525000, "rew": 83.3068, "rew_std": 29.8053, "Agent": "TD3"}, {"env_step": 530000, "rew": 80.923, "rew_std": 29.902, "Agent": "TD3"}, {"env_step": 535000, "rew": 82.1793, "rew_std": 31.9273, "Agent": "TD3"}, {"env_step": 540000, "rew": 81.4991, "rew_std": 31.5799, "Agent": "TD3"}, {"env_step": 545000, "rew": 79.6794, "rew_std": 30.9146, "Agent": "TD3"}, {"env_step": 550000, "rew": 81.2391, "rew_std": 29.9395, "Agent": "TD3"}, {"env_step": 555000, "rew": 81.0082, "rew_std": 30.8059, "Agent": "TD3"}, {"env_step": 560000, "rew": 82.1303, "rew_std": 31.0288, "Agent": "TD3"}, {"env_step": 565000, "rew": 82.3656, "rew_std": 32.1176, "Agent": "TD3"}, {"env_step": 570000, "rew": 81.7197, "rew_std": 32.3548, "Agent": "TD3"}, {"env_step": 575000, "rew": 80.6937, "rew_std": 30.7503, "Agent": "TD3"}, {"env_step": 580000, "rew": 84.4082, "rew_std": 30.6736, "Agent": "TD3"}, {"env_step": 585000, "rew": 82.4516, "rew_std": 30.6142, "Agent": "TD3"}, {"env_step": 590000, "rew": 86.4441, "rew_std": 30.2415, "Agent": "TD3"}, {"env_step": 595000, "rew": 81.5008, "rew_std": 29.9591, "Agent": "TD3"}, {"env_step": 600000, "rew": 84.318, "rew_std": 29.7351, "Agent": "TD3"}, {"env_step": 605000, "rew": 83.5531, "rew_std": 30.6785, "Agent": "TD3"}, {"env_step": 610000, "rew": 86.2638, "rew_std": 30.767, "Agent": "TD3"}, {"env_step": 615000, "rew": 87.2233, "rew_std": 30.9616, "Agent": "TD3"}, {"env_step": 620000, "rew": 85.1809, "rew_std": 31.0423, "Agent": "TD3"}, {"env_step": 625000, "rew": 88.4356, "rew_std": 31.6306, "Agent": "TD3"}, {"env_step": 630000, "rew": 82.1748, "rew_std": 32.4394, "Agent": "TD3"}, {"env_step": 635000, "rew": 87.2665, "rew_std": 32.4218, "Agent": "TD3"}, {"env_step": 640000, "rew": 85.785, "rew_std": 30.6537, "Agent": "TD3"}, {"env_step": 645000, "rew": 86.5869, "rew_std": 30.4763, "Agent": "TD3"}, {"env_step": 650000, "rew": 87.3203, "rew_std": 32.0232, "Agent": "TD3"}, {"env_step": 655000, "rew": 87.7947, "rew_std": 29.9599, "Agent": "TD3"}, {"env_step": 660000, "rew": 87.8461, "rew_std": 30.2639, "Agent": "TD3"}, {"env_step": 665000, "rew": 88.8311, "rew_std": 32.377, "Agent": "TD3"}, {"env_step": 670000, "rew": 84.6939, "rew_std": 32.3997, "Agent": "TD3"}, {"env_step": 675000, "rew": 85.5088, "rew_std": 32.55, "Agent": "TD3"}, {"env_step": 680000, "rew": 87.4098, "rew_std": 31.2783, "Agent": "TD3"}, {"env_step": 685000, "rew": 88.276, "rew_std": 32.8023, "Agent": "TD3"}, {"env_step": 690000, "rew": 87.2682, "rew_std": 32.3965, "Agent": "TD3"}, {"env_step": 695000, "rew": 89.4875, "rew_std": 31.3106, "Agent": "TD3"}, {"env_step": 700000, "rew": 89.3074, "rew_std": 31.7229, "Agent": "TD3"}, {"env_step": 705000, "rew": 91.8734, "rew_std": 31.0277, "Agent": "TD3"}, {"env_step": 710000, "rew": 88.4577, "rew_std": 28.9902, "Agent": "TD3"}, {"env_step": 715000, "rew": 90.7966, "rew_std": 28.9473, "Agent": "TD3"}, {"env_step": 720000, "rew": 88.8606, "rew_std": 29.6708, "Agent": "TD3"}, {"env_step": 725000, "rew": 90.5793, "rew_std": 30.4509, "Agent": "TD3"}, {"env_step": 730000, "rew": 92.6566, "rew_std": 29.1638, "Agent": "TD3"}, {"env_step": 735000, "rew": 91.0644, "rew_std": 30.132, "Agent": "TD3"}, {"env_step": 740000, "rew": 93.81, "rew_std": 31.2251, "Agent": "TD3"}, {"env_step": 745000, "rew": 93.186, "rew_std": 30.4468, "Agent": "TD3"}, {"env_step": 750000, "rew": 92.6643, "rew_std": 31.8271, "Agent": "TD3"}, {"env_step": 755000, "rew": 95.9085, "rew_std": 30.7396, "Agent": "TD3"}, {"env_step": 760000, "rew": 93.2766, "rew_std": 30.5946, "Agent": "TD3"}, {"env_step": 765000, "rew": 91.7779, "rew_std": 30.9428, "Agent": "TD3"}, {"env_step": 770000, "rew": 92.8975, "rew_std": 30.3727, "Agent": "TD3"}, {"env_step": 775000, "rew": 93.2678, "rew_std": 30.035, "Agent": "TD3"}, {"env_step": 780000, "rew": 94.9334, "rew_std": 30.218, "Agent": "TD3"}, {"env_step": 785000, "rew": 97.689, "rew_std": 33.428, "Agent": "TD3"}, {"env_step": 790000, "rew": 95.8544, "rew_std": 29.5172, "Agent": "TD3"}, {"env_step": 795000, "rew": 95.3164, "rew_std": 31.1582, "Agent": "TD3"}, {"env_step": 800000, "rew": 99.102, "rew_std": 34.4922, "Agent": "TD3"}, {"env_step": 805000, "rew": 98.9969, "rew_std": 33.5135, "Agent": "TD3"}, {"env_step": 810000, "rew": 97.6485, "rew_std": 34.9215, "Agent": "TD3"}, {"env_step": 815000, "rew": 97.5072, "rew_std": 33.132, "Agent": "TD3"}, {"env_step": 820000, "rew": 100.9863, "rew_std": 32.6827, "Agent": "TD3"}, {"env_step": 825000, "rew": 97.8096, "rew_std": 31.934, "Agent": "TD3"}, {"env_step": 830000, "rew": 100.3499, "rew_std": 33.2398, "Agent": "TD3"}, {"env_step": 835000, "rew": 99.6387, "rew_std": 32.1877, "Agent": "TD3"}, {"env_step": 840000, "rew": 100.1878, "rew_std": 35.5064, "Agent": "TD3"}, {"env_step": 845000, "rew": 100.9504, "rew_std": 31.808, "Agent": "TD3"}, {"env_step": 850000, "rew": 102.5165, "rew_std": 34.9759, "Agent": "TD3"}, {"env_step": 855000, "rew": 99.8103, "rew_std": 32.3758, "Agent": "TD3"}, {"env_step": 860000, "rew": 101.7581, "rew_std": 31.9818, "Agent": "TD3"}, {"env_step": 865000, "rew": 100.7405, "rew_std": 32.7885, "Agent": "TD3"}, {"env_step": 870000, "rew": 100.8249, "rew_std": 33.4003, "Agent": "TD3"}, {"env_step": 875000, "rew": 100.7485, "rew_std": 33.2082, "Agent": "TD3"}, {"env_step": 880000, "rew": 102.5295, "rew_std": 34.4189, "Agent": "TD3"}, {"env_step": 885000, "rew": 99.8226, "rew_std": 32.9092, "Agent": "TD3"}, {"env_step": 890000, "rew": 94.6528, "rew_std": 34.0478, "Agent": "TD3"}, {"env_step": 895000, "rew": 93.6971, "rew_std": 32.7561, "Agent": "TD3"}, {"env_step": 900000, "rew": 101.2272, "rew_std": 33.0066, "Agent": "TD3"}, {"env_step": 905000, "rew": 99.1686, "rew_std": 32.928, "Agent": "TD3"}, {"env_step": 910000, "rew": 101.3434, "rew_std": 33.0791, "Agent": "TD3"}, {"env_step": 915000, "rew": 100.4035, "rew_std": 33.5787, "Agent": "TD3"}, {"env_step": 920000, "rew": 99.3797, "rew_std": 34.4315, "Agent": "TD3"}, {"env_step": 925000, "rew": 100.1444, "rew_std": 34.0045, "Agent": "TD3"}, {"env_step": 930000, "rew": 97.4632, "rew_std": 33.0392, "Agent": "TD3"}, {"env_step": 935000, "rew": 101.2141, "rew_std": 32.8136, "Agent": "TD3"}, {"env_step": 940000, "rew": 97.8222, "rew_std": 34.4894, "Agent": "TD3"}, {"env_step": 945000, "rew": 99.0311, "rew_std": 31.3895, "Agent": "TD3"}, {"env_step": 950000, "rew": 99.9747, "rew_std": 32.4121, "Agent": "TD3"}, {"env_step": 955000, "rew": 98.6989, "rew_std": 33.4333, "Agent": "TD3"}, {"env_step": 960000, "rew": 91.8445, "rew_std": 36.9984, "Agent": "TD3"}, {"env_step": 965000, "rew": 93.2391, "rew_std": 35.273, "Agent": "TD3"}, {"env_step": 970000, "rew": 93.917, "rew_std": 33.1432, "Agent": "TD3"}, {"env_step": 975000, "rew": 99.1082, "rew_std": 31.1777, "Agent": "TD3"}, {"env_step": 980000, "rew": 99.4874, "rew_std": 34.1046, "Agent": "TD3"}, {"env_step": 985000, "rew": 98.5446, "rew_std": 33.182, "Agent": "TD3"}, {"env_step": 990000, "rew": 104.1637, "rew_std": 34.1874, "Agent": "TD3"}, {"env_step": 995000, "rew": 101.2461, "rew_std": 34.129, "Agent": "TD3"}, {"env_step": 1000000, "rew": 96.2436, "rew_std": 33.4562, "Agent": "TD3"}, {"env_step": 1005000, "rew": 96.9054, "rew_std": 34.0766, "Agent": "TD3"}, {"env_step": 1010000, "rew": 102.1064, "rew_std": 32.5263, "Agent": "TD3"}, {"env_step": 1015000, "rew": 100.9795, "rew_std": 34.6542, "Agent": "TD3"}, {"env_step": 1020000, "rew": 98.552, "rew_std": 32.6727, "Agent": "TD3"}, {"env_step": 1025000, "rew": 100.4183, "rew_std": 35.4292, "Agent": "TD3"}, {"env_step": 1030000, "rew": 101.5891, "rew_std": 34.5836, "Agent": "TD3"}, {"env_step": 1035000, "rew": 101.3104, "rew_std": 33.444, "Agent": "TD3"}, {"env_step": 1040000, "rew": 98.8935, "rew_std": 34.7818, "Agent": "TD3"}, {"env_step": 1045000, "rew": 100.1006, "rew_std": 34.472, "Agent": "TD3"}, {"env_step": 1050000, "rew": 101.8728, "rew_std": 33.0307, "Agent": "TD3"}, {"env_step": 1055000, "rew": 100.3289, "rew_std": 33.8025, "Agent": "TD3"}, {"env_step": 1060000, "rew": 99.3667, "rew_std": 34.6098, "Agent": "TD3"}, {"env_step": 1065000, "rew": 100.904, "rew_std": 33.4198, "Agent": "TD3"}, {"env_step": 1070000, "rew": 101.9024, "rew_std": 33.8339, "Agent": "TD3"}, {"env_step": 1075000, "rew": 102.4069, "rew_std": 34.2427, "Agent": "TD3"}, {"env_step": 1080000, "rew": 100.5603, "rew_std": 33.2086, "Agent": "TD3"}, {"env_step": 1085000, "rew": 104.1487, "rew_std": 33.6152, "Agent": "TD3"}, {"env_step": 1090000, "rew": 101.7053, "rew_std": 33.0356, "Agent": "TD3"}, {"env_step": 1095000, "rew": 102.106, "rew_std": 32.8048, "Agent": "TD3"}, {"env_step": 1100000, "rew": 101.0828, "rew_std": 33.3971, "Agent": "TD3"}, {"env_step": 1105000, "rew": 102.5127, "rew_std": 32.3886, "Agent": "TD3"}, {"env_step": 1110000, "rew": 95.4044, "rew_std": 31.6599, "Agent": "TD3"}, {"env_step": 1115000, "rew": 102.8439, "rew_std": 32.6782, "Agent": "TD3"}, {"env_step": 1120000, "rew": 102.2217, "rew_std": 35.2064, "Agent": "TD3"}, {"env_step": 1125000, "rew": 102.1562, "rew_std": 34.7002, "Agent": "TD3"}, {"env_step": 1130000, "rew": 101.2998, "rew_std": 35.3869, "Agent": "TD3"}, {"env_step": 1135000, "rew": 102.9873, "rew_std": 32.7296, "Agent": "TD3"}, {"env_step": 1140000, "rew": 102.4587, "rew_std": 33.3457, "Agent": "TD3"}, {"env_step": 1145000, "rew": 103.994, "rew_std": 32.6864, "Agent": "TD3"}, {"env_step": 1150000, "rew": 103.5746, "rew_std": 31.4598, "Agent": "TD3"}, {"env_step": 1155000, "rew": 102.734, "rew_std": 31.8162, "Agent": "TD3"}, {"env_step": 1160000, "rew": 106.0746, "rew_std": 31.9923, "Agent": "TD3"}, {"env_step": 1165000, "rew": 104.1096, "rew_std": 32.0328, "Agent": "TD3"}, {"env_step": 1170000, "rew": 102.8255, "rew_std": 34.3626, "Agent": "TD3"}, {"env_step": 1175000, "rew": 102.9476, "rew_std": 33.3184, "Agent": "TD3"}, {"env_step": 1180000, "rew": 103.5396, "rew_std": 32.3704, "Agent": "TD3"}, {"env_step": 1185000, "rew": 103.7538, "rew_std": 33.86, "Agent": "TD3"}, {"env_step": 1190000, "rew": 106.8142, "rew_std": 32.7257, "Agent": "TD3"}, {"env_step": 1195000, "rew": 104.5737, "rew_std": 32.7523, "Agent": "TD3"}, {"env_step": 1200000, "rew": 103.5237, "rew_std": 33.3121, "Agent": "TD3"}, {"env_step": 1205000, "rew": 104.1739, "rew_std": 33.8102, "Agent": "TD3"}, {"env_step": 1210000, "rew": 106.0291, "rew_std": 33.2563, "Agent": "TD3"}, {"env_step": 1215000, "rew": 106.5229, "rew_std": 34.5558, "Agent": "TD3"}, {"env_step": 1220000, "rew": 108.357, "rew_std": 33.9605, "Agent": "TD3"}, {"env_step": 1225000, "rew": 104.495, "rew_std": 33.2559, "Agent": "TD3"}, {"env_step": 1230000, "rew": 107.376, "rew_std": 32.6799, "Agent": "TD3"}, {"env_step": 1235000, "rew": 105.9014, "rew_std": 33.3857, "Agent": "TD3"}, {"env_step": 1240000, "rew": 106.8905, "rew_std": 33.261, "Agent": "TD3"}, {"env_step": 1245000, "rew": 108.226, "rew_std": 32.7489, "Agent": "TD3"}, {"env_step": 1250000, "rew": 106.2262, "rew_std": 34.3315, "Agent": "TD3"}, {"env_step": 0, "rew": -46.1947, "rew_std": 25.5804, "Agent": "TRPO"}, {"env_step": 30720, "rew": -13.9012, "rew_std": 11.913, "Agent": "TRPO"}, {"env_step": 61440, "rew": 18.0615, "rew_std": 9.0166, "Agent": "TRPO"}, {"env_step": 92160, "rew": 51.9175, "rew_std": 16.4955, "Agent": "TRPO"}, {"env_step": 122880, "rew": 108.4958, "rew_std": 33.2241, "Agent": "TRPO"}, {"env_step": 153600, "rew": 183.9944, "rew_std": 35.8262, "Agent": "TRPO"}, {"env_step": 184320, "rew": 257.4861, "rew_std": 60.051, "Agent": "TRPO"}, {"env_step": 215040, "rew": 430.4068, "rew_std": 124.2541, "Agent": "TRPO"}, {"env_step": 245760, "rew": 564.2017, "rew_std": 146.0606, "Agent": "TRPO"}, {"env_step": 276480, "rew": 682.5168, "rew_std": 122.3187, "Agent": "TRPO"}, {"env_step": 307200, "rew": 832.5033, "rew_std": 159.2968, "Agent": "TRPO"}, {"env_step": 337920, "rew": 1014.1049, "rew_std": 221.8108, "Agent": "TRPO"}, {"env_step": 368640, "rew": 1128.5618, "rew_std": 167.9213, "Agent": "TRPO"}, {"env_step": 399360, "rew": 1146.4494, "rew_std": 215.1602, "Agent": "TRPO"}, {"env_step": 430080, "rew": 1233.6374, "rew_std": 286.8604, "Agent": "TRPO"}, {"env_step": 460800, "rew": 1261.4876, "rew_std": 301.0892, "Agent": "TRPO"}, {"env_step": 491520, "rew": 1472.6865, "rew_std": 215.7148, "Agent": "TRPO"}, {"env_step": 522240, "rew": 1501.5343, "rew_std": 428.1053, "Agent": "TRPO"}, {"env_step": 552960, "rew": 1631.5161, "rew_std": 558.1692, "Agent": "TRPO"}, {"env_step": 583680, "rew": 1741.6005, "rew_std": 483.7599, "Agent": "TRPO"}, {"env_step": 614400, "rew": 1809.3667, "rew_std": 444.8329, "Agent": "TRPO"}, {"env_step": 645120, "rew": 1943.2351, "rew_std": 386.0455, "Agent": "TRPO"}, {"env_step": 675840, "rew": 2119.3593, "rew_std": 463.9853, "Agent": "TRPO"}, {"env_step": 706560, "rew": 2160.9349, "rew_std": 299.8534, "Agent": "TRPO"}, {"env_step": 737280, "rew": 2205.6584, "rew_std": 495.8226, "Agent": "TRPO"}, {"env_step": 768000, "rew": 2148.4616, "rew_std": 617.4263, "Agent": "TRPO"}, {"env_step": 798720, "rew": 2294.2347, "rew_std": 619.654, "Agent": "TRPO"}, {"env_step": 829440, "rew": 2334.3877, "rew_std": 569.8366, "Agent": "TRPO"}, {"env_step": 860160, "rew": 2373.0195, "rew_std": 635.4872, "Agent": "TRPO"}, {"env_step": 890880, "rew": 2572.1874, "rew_std": 771.2781, "Agent": "TRPO"}, {"env_step": 921600, "rew": 2568.2581, "rew_std": 539.7726, "Agent": "TRPO"}, {"env_step": 952320, "rew": 2639.2867, "rew_std": 572.4301, "Agent": "TRPO"}, {"env_step": 983040, "rew": 2866.6508, "rew_std": 707.8937, "Agent": "TRPO"}, {"env_step": 1013760, "rew": 2731.0697, "rew_std": 675.644, "Agent": "TRPO"}, {"env_step": 1044480, "rew": 2702.1965, "rew_std": 742.4807, "Agent": "TRPO"}, {"env_step": 1075200, "rew": 3001.7621, "rew_std": 688.8314, "Agent": "TRPO"}, {"env_step": 1105920, "rew": 2877.8716, "rew_std": 922.2411, "Agent": "TRPO"}, {"env_step": 1136640, "rew": 3104.9072, "rew_std": 800.8227, "Agent": "TRPO"}, {"env_step": 1167360, "rew": 2976.5528, "rew_std": 661.5153, "Agent": "TRPO"}, {"env_step": 1198080, "rew": 3196.2228, "rew_std": 707.9679, "Agent": "TRPO"}, {"env_step": 1228800, "rew": 3237.2005, "rew_std": 505.6685, "Agent": "TRPO"}, {"env_step": 1259520, "rew": 3098.3083, "rew_std": 578.4346, "Agent": "TRPO"}, {"env_step": 1290240, "rew": 3356.962, "rew_std": 786.3838, "Agent": "TRPO"}, {"env_step": 1320960, "rew": 3241.2647, "rew_std": 600.3495, "Agent": "TRPO"}, {"env_step": 1351680, "rew": 3361.1722, "rew_std": 547.5151, "Agent": "TRPO"}, {"env_step": 1382400, "rew": 3495.3879, "rew_std": 662.5387, "Agent": "TRPO"}, {"env_step": 1413120, "rew": 3388.8443, "rew_std": 537.1016, "Agent": "TRPO"}, {"env_step": 1443840, "rew": 3493.6456, "rew_std": 509.2391, "Agent": "TRPO"}, {"env_step": 1474560, "rew": 3256.3625, "rew_std": 412.1903, "Agent": "TRPO"}, {"env_step": 1505280, "rew": 3695.3862, "rew_std": 644.5284, "Agent": "TRPO"}, {"env_step": 1536000, "rew": 3685.0253, "rew_std": 537.4812, "Agent": "TRPO"}, {"env_step": 1566720, "rew": 3565.3736, "rew_std": 647.2313, "Agent": "TRPO"}, {"env_step": 1597440, "rew": 3790.5153, "rew_std": 702.1943, "Agent": "TRPO"}, {"env_step": 1628160, "rew": 3598.3267, "rew_std": 626.2541, "Agent": "TRPO"}, {"env_step": 1658880, "rew": 3970.7287, "rew_std": 647.9836, "Agent": "TRPO"}, {"env_step": 1689600, "rew": 3658.6492, "rew_std": 645.0697, "Agent": "TRPO"}, {"env_step": 1720320, "rew": 3848.0345, "rew_std": 701.1273, "Agent": "TRPO"}, {"env_step": 1751040, "rew": 4061.4513, "rew_std": 690.0289, "Agent": "TRPO"}, {"env_step": 1781760, "rew": 3807.921, "rew_std": 706.5999, "Agent": "TRPO"}, {"env_step": 1812480, "rew": 3802.0342, "rew_std": 663.7139, "Agent": "TRPO"}, {"env_step": 1843200, "rew": 3873.6662, "rew_std": 566.3833, "Agent": "TRPO"}, {"env_step": 1873920, "rew": 4038.6082, "rew_std": 518.5079, "Agent": "TRPO"}, {"env_step": 1904640, "rew": 4167.3221, "rew_std": 440.9658, "Agent": "TRPO"}, {"env_step": 1935360, "rew": 3922.2855, "rew_std": 733.8238, "Agent": "TRPO"}, {"env_step": 1966080, "rew": 4174.7307, "rew_std": 495.1849, "Agent": "TRPO"}, {"env_step": 1996800, "rew": 4155.8384, "rew_std": 477.6376, "Agent": "TRPO"}, {"env_step": 2027520, "rew": 4259.3055, "rew_std": 577.3636, "Agent": "TRPO"}, {"env_step": 2058240, "rew": 4256.1933, "rew_std": 594.2826, "Agent": "TRPO"}, {"env_step": 2088960, "rew": 4355.0669, "rew_std": 605.8978, "Agent": "TRPO"}, {"env_step": 2119680, "rew": 4297.2252, "rew_std": 474.421, "Agent": "TRPO"}, {"env_step": 2150400, "rew": 4287.4698, "rew_std": 363.7332, "Agent": "TRPO"}, {"env_step": 2181120, "rew": 4398.2137, "rew_std": 354.3603, "Agent": "TRPO"}, {"env_step": 2211840, "rew": 4413.5796, "rew_std": 460.9681, "Agent": "TRPO"}, {"env_step": 2242560, "rew": 4329.001, "rew_std": 530.2755, "Agent": "TRPO"}, {"env_step": 2273280, "rew": 4252.3464, "rew_std": 597.5882, "Agent": "TRPO"}, {"env_step": 2304000, "rew": 4462.5657, "rew_std": 495.3802, "Agent": "TRPO"}, {"env_step": 2334720, "rew": 4374.2838, "rew_std": 457.0956, "Agent": "TRPO"}, {"env_step": 2365440, "rew": 4529.4549, "rew_std": 461.1733, "Agent": "TRPO"}, {"env_step": 2396160, "rew": 4291.4468, "rew_std": 411.2142, "Agent": "TRPO"}, {"env_step": 2426880, "rew": 4430.1459, "rew_std": 601.8812, "Agent": "TRPO"}, {"env_step": 2457600, "rew": 4508.078, "rew_std": 465.0507, "Agent": "TRPO"}, {"env_step": 2488320, "rew": 4151.3796, "rew_std": 414.656, "Agent": "TRPO"}, {"env_step": 2519040, "rew": 4705.7212, "rew_std": 477.1206, "Agent": "TRPO"}, {"env_step": 2549760, "rew": 4641.3111, "rew_std": 622.6369, "Agent": "TRPO"}, {"env_step": 2580480, "rew": 4484.1452, "rew_std": 442.0811, "Agent": "TRPO"}, {"env_step": 2611200, "rew": 4658.7294, "rew_std": 432.1496, "Agent": "TRPO"}, {"env_step": 2641920, "rew": 4625.8785, "rew_std": 357.4208, "Agent": "TRPO"}, {"env_step": 2672640, "rew": 4284.9794, "rew_std": 599.7262, "Agent": "TRPO"}, {"env_step": 2703360, "rew": 4419.1972, "rew_std": 447.5325, "Agent": "TRPO"}, {"env_step": 2734080, "rew": 4746.1067, "rew_std": 580.4515, "Agent": "TRPO"}, {"env_step": 2764800, "rew": 4756.6644, "rew_std": 502.4658, "Agent": "TRPO"}, {"env_step": 2795520, "rew": 4607.7287, "rew_std": 427.1984, "Agent": "TRPO"}, {"env_step": 2826240, "rew": 4404.4363, "rew_std": 555.3297, "Agent": "TRPO"}, {"env_step": 2856960, "rew": 4816.8623, "rew_std": 528.6794, "Agent": "TRPO"}, {"env_step": 2887680, "rew": 4753.3578, "rew_std": 715.7057, "Agent": "TRPO"}, {"env_step": 2918400, "rew": 4604.763, "rew_std": 447.4458, "Agent": "TRPO"}, {"env_step": 2949120, "rew": 4800.2693, "rew_std": 602.9667, "Agent": "TRPO"}, {"env_step": 2979840, "rew": 4772.9396, "rew_std": 368.1691, "Agent": "TRPO"}, {"env_step": 3010560, "rew": 4897.0257, "rew_std": 449.1329, "Agent": "TRPO"}, {"env_step": 3041280, "rew": 4841.7814, "rew_std": 534.8951, "Agent": "TRPO"}, {"env_step": 3072000, "rew": 4604.9043, "rew_std": 580.976, "Agent": "TRPO"}, {"env_step": 0, "rew": -46.1947, "rew_std": 25.5804, "Agent": "NPG"}, {"env_step": 30720, "rew": -29.0009, "rew_std": 15.8916, "Agent": "NPG"}, {"env_step": 61440, "rew": -10.4279, "rew_std": 11.0242, "Agent": "NPG"}, {"env_step": 92160, "rew": 11.1187, "rew_std": 9.0737, "Agent": "NPG"}, {"env_step": 122880, "rew": 27.5852, "rew_std": 16.4638, "Agent": "NPG"}, {"env_step": 153600, "rew": 56.7285, "rew_std": 23.8642, "Agent": "NPG"}, {"env_step": 184320, "rew": 81.2319, "rew_std": 29.1525, "Agent": "NPG"}, {"env_step": 215040, "rew": 159.4704, "rew_std": 47.3793, "Agent": "NPG"}, {"env_step": 245760, "rew": 174.9781, "rew_std": 57.693, "Agent": "NPG"}, {"env_step": 276480, "rew": 178.6358, "rew_std": 32.807, "Agent": "NPG"}, {"env_step": 307200, "rew": 218.3212, "rew_std": 41.035, "Agent": "NPG"}, {"env_step": 337920, "rew": 368.663, "rew_std": 80.0623, "Agent": "NPG"}, {"env_step": 368640, "rew": 432.2163, "rew_std": 63.1225, "Agent": "NPG"}, {"env_step": 399360, "rew": 568.0807, "rew_std": 125.517, "Agent": "NPG"}, {"env_step": 430080, "rew": 631.1259, "rew_std": 114.6371, "Agent": "NPG"}, {"env_step": 460800, "rew": 753.5929, "rew_std": 118.5185, "Agent": "NPG"}, {"env_step": 491520, "rew": 864.7371, "rew_std": 183.4071, "Agent": "NPG"}, {"env_step": 522240, "rew": 940.2978, "rew_std": 158.8018, "Agent": "NPG"}, {"env_step": 552960, "rew": 1021.7661, "rew_std": 159.2406, "Agent": "NPG"}, {"env_step": 583680, "rew": 1131.714, "rew_std": 150.4756, "Agent": "NPG"}, {"env_step": 614400, "rew": 1201.5181, "rew_std": 164.258, "Agent": "NPG"}, {"env_step": 645120, "rew": 1343.1191, "rew_std": 184.6686, "Agent": "NPG"}, {"env_step": 675840, "rew": 1318.401, "rew_std": 229.7468, "Agent": "NPG"}, {"env_step": 706560, "rew": 1459.1159, "rew_std": 222.6298, "Agent": "NPG"}, {"env_step": 737280, "rew": 1542.0295, "rew_std": 203.9479, "Agent": "NPG"}, {"env_step": 768000, "rew": 1634.9176, "rew_std": 218.2131, "Agent": "NPG"}, {"env_step": 798720, "rew": 1692.0983, "rew_std": 343.9785, "Agent": "NPG"}, {"env_step": 829440, "rew": 1906.1507, "rew_std": 457.5102, "Agent": "NPG"}, {"env_step": 860160, "rew": 1775.1748, "rew_std": 342.0144, "Agent": "NPG"}, {"env_step": 890880, "rew": 2102.0494, "rew_std": 355.8503, "Agent": "NPG"}, {"env_step": 921600, "rew": 2046.7794, "rew_std": 340.8884, "Agent": "NPG"}, {"env_step": 952320, "rew": 2164.3512, "rew_std": 405.7482, "Agent": "NPG"}, {"env_step": 983040, "rew": 2348.312, "rew_std": 509.9401, "Agent": "NPG"}, {"env_step": 1013760, "rew": 2365.8958, "rew_std": 523.6603, "Agent": "NPG"}, {"env_step": 1044480, "rew": 2324.9164, "rew_std": 528.4526, "Agent": "NPG"}, {"env_step": 1075200, "rew": 2493.722, "rew_std": 583.8895, "Agent": "NPG"}, {"env_step": 1105920, "rew": 2638.7282, "rew_std": 650.3356, "Agent": "NPG"}, {"env_step": 1136640, "rew": 2588.3598, "rew_std": 506.4239, "Agent": "NPG"}, {"env_step": 1167360, "rew": 2657.8218, "rew_std": 542.1361, "Agent": "NPG"}, {"env_step": 1198080, "rew": 2772.0887, "rew_std": 399.8722, "Agent": "NPG"}, {"env_step": 1228800, "rew": 2983.8532, "rew_std": 405.1427, "Agent": "NPG"}, {"env_step": 1259520, "rew": 2969.912, "rew_std": 543.0224, "Agent": "NPG"}, {"env_step": 1290240, "rew": 2898.5856, "rew_std": 687.1927, "Agent": "NPG"}, {"env_step": 1320960, "rew": 2979.687, "rew_std": 685.3181, "Agent": "NPG"}, {"env_step": 1351680, "rew": 2937.7865, "rew_std": 654.2811, "Agent": "NPG"}, {"env_step": 1382400, "rew": 3180.3262, "rew_std": 502.1675, "Agent": "NPG"}, {"env_step": 1413120, "rew": 3181.4746, "rew_std": 526.9918, "Agent": "NPG"}, {"env_step": 1443840, "rew": 3186.725, "rew_std": 483.0104, "Agent": "NPG"}, {"env_step": 1474560, "rew": 3347.1624, "rew_std": 474.3212, "Agent": "NPG"}, {"env_step": 1505280, "rew": 3264.1247, "rew_std": 542.1326, "Agent": "NPG"}, {"env_step": 1536000, "rew": 3267.3737, "rew_std": 536.477, "Agent": "NPG"}, {"env_step": 1566720, "rew": 3677.709, "rew_std": 486.0857, "Agent": "NPG"}, {"env_step": 1597440, "rew": 3379.1083, "rew_std": 683.1763, "Agent": "NPG"}, {"env_step": 1628160, "rew": 3533.6653, "rew_std": 437.9623, "Agent": "NPG"}, {"env_step": 1658880, "rew": 3534.0292, "rew_std": 787.6867, "Agent": "NPG"}, {"env_step": 1689600, "rew": 3399.7589, "rew_std": 685.1419, "Agent": "NPG"}, {"env_step": 1720320, "rew": 3587.7024, "rew_std": 471.134, "Agent": "NPG"}, {"env_step": 1751040, "rew": 3560.6233, "rew_std": 616.8093, "Agent": "NPG"}, {"env_step": 1781760, "rew": 3635.7303, "rew_std": 608.4666, "Agent": "NPG"}, {"env_step": 1812480, "rew": 3713.5899, "rew_std": 802.8577, "Agent": "NPG"}, {"env_step": 1843200, "rew": 3590.908, "rew_std": 557.7779, "Agent": "NPG"}, {"env_step": 1873920, "rew": 3903.7812, "rew_std": 655.7805, "Agent": "NPG"}, {"env_step": 1904640, "rew": 3770.5075, "rew_std": 645.9229, "Agent": "NPG"}, {"env_step": 1935360, "rew": 4110.3599, "rew_std": 657.3085, "Agent": "NPG"}, {"env_step": 1966080, "rew": 3851.8914, "rew_std": 745.2079, "Agent": "NPG"}, {"env_step": 1996800, "rew": 3998.0014, "rew_std": 650.7886, "Agent": "NPG"}, {"env_step": 2027520, "rew": 3739.3657, "rew_std": 715.6727, "Agent": "NPG"}, {"env_step": 2058240, "rew": 4230.5764, "rew_std": 578.5901, "Agent": "NPG"}, {"env_step": 2088960, "rew": 4140.935, "rew_std": 624.3245, "Agent": "NPG"}, {"env_step": 2119680, "rew": 4228.9808, "rew_std": 573.9637, "Agent": "NPG"}, {"env_step": 2150400, "rew": 4142.6649, "rew_std": 419.5781, "Agent": "NPG"}, {"env_step": 2181120, "rew": 4089.8147, "rew_std": 302.9776, "Agent": "NPG"}, {"env_step": 2211840, "rew": 4276.8854, "rew_std": 674.4424, "Agent": "NPG"}, {"env_step": 2242560, "rew": 4312.1654, "rew_std": 567.5481, "Agent": "NPG"}, {"env_step": 2273280, "rew": 4224.3566, "rew_std": 445.0954, "Agent": "NPG"}, {"env_step": 2304000, "rew": 3878.7632, "rew_std": 866.3551, "Agent": "NPG"}, {"env_step": 2334720, "rew": 4001.3801, "rew_std": 478.1705, "Agent": "NPG"}, {"env_step": 2365440, "rew": 4284.5594, "rew_std": 448.8121, "Agent": "NPG"}, {"env_step": 2396160, "rew": 4406.6923, "rew_std": 516.8683, "Agent": "NPG"}, {"env_step": 2426880, "rew": 4145.9032, "rew_std": 519.4268, "Agent": "NPG"}, {"env_step": 2457600, "rew": 4241.7632, "rew_std": 685.6791, "Agent": "NPG"}, {"env_step": 2488320, "rew": 4154.1219, "rew_std": 437.1567, "Agent": "NPG"}, {"env_step": 2519040, "rew": 4495.7374, "rew_std": 449.1806, "Agent": "NPG"}, {"env_step": 2549760, "rew": 4289.7487, "rew_std": 526.7084, "Agent": "NPG"}, {"env_step": 2580480, "rew": 4246.0211, "rew_std": 530.6265, "Agent": "NPG"}, {"env_step": 2611200, "rew": 4463.8985, "rew_std": 572.8797, "Agent": "NPG"}, {"env_step": 2641920, "rew": 4298.6282, "rew_std": 665.9416, "Agent": "NPG"}, {"env_step": 2672640, "rew": 4485.5394, "rew_std": 359.9158, "Agent": "NPG"}, {"env_step": 2703360, "rew": 4820.597, "rew_std": 318.5627, "Agent": "NPG"}, {"env_step": 2734080, "rew": 4540.1727, "rew_std": 412.9545, "Agent": "NPG"}, {"env_step": 2764800, "rew": 4500.1939, "rew_std": 596.1343, "Agent": "NPG"}, {"env_step": 2795520, "rew": 4664.0692, "rew_std": 426.6286, "Agent": "NPG"}, {"env_step": 2826240, "rew": 4516.467, "rew_std": 435.656, "Agent": "NPG"}, {"env_step": 2856960, "rew": 4545.139, "rew_std": 496.411, "Agent": "NPG"}, {"env_step": 2887680, "rew": 4444.0518, "rew_std": 596.6213, "Agent": "NPG"}, {"env_step": 2918400, "rew": 4658.8307, "rew_std": 652.0561, "Agent": "NPG"}, {"env_step": 2949120, "rew": 4509.4439, "rew_std": 466.3483, "Agent": "NPG"}, {"env_step": 2979840, "rew": 4589.8375, "rew_std": 414.0363, "Agent": "NPG"}, {"env_step": 3010560, "rew": 4693.0902, "rew_std": 490.4674, "Agent": "NPG"}, {"env_step": 3041280, "rew": 4678.8152, "rew_std": 313.0054, "Agent": "NPG"}, {"env_step": 3072000, "rew": 4736.6273, "rew_std": 376.7205, "Agent": "NPG"}, {"env_step": 0, "rew": 1.0903, "rew_std": 3.5978, "Agent": "PPO"}, {"env_step": 30720, "rew": 14.541, "rew_std": 5.4261, "Agent": "PPO"}, {"env_step": 61440, "rew": -17.2246, "rew_std": 3.7694, "Agent": "PPO"}, {"env_step": 92160, "rew": -11.2966, "rew_std": 8.1654, "Agent": "PPO"}, {"env_step": 122880, "rew": -10.0344, "rew_std": 10.6224, "Agent": "PPO"}, {"env_step": 153600, "rew": 3.5313, "rew_std": 14.4579, "Agent": "PPO"}, {"env_step": 184320, "rew": 3.9484, "rew_std": 15.9274, "Agent": "PPO"}, {"env_step": 215040, "rew": 12.3234, "rew_std": 13.8123, "Agent": "PPO"}, {"env_step": 245760, "rew": 10.885, "rew_std": 19.2383, "Agent": "PPO"}, {"env_step": 276480, "rew": 15.0975, "rew_std": 14.2842, "Agent": "PPO"}, {"env_step": 307200, "rew": 16.7842, "rew_std": 17.8373, "Agent": "PPO"}, {"env_step": 337920, "rew": 24.768, "rew_std": 20.6693, "Agent": "PPO"}, {"env_step": 368640, "rew": 28.4681, "rew_std": 16.759, "Agent": "PPO"}, {"env_step": 399360, "rew": 31.9449, "rew_std": 14.1058, "Agent": "PPO"}, {"env_step": 430080, "rew": 33.7103, "rew_std": 8.9838, "Agent": "PPO"}, {"env_step": 460800, "rew": 31.834, "rew_std": 14.3058, "Agent": "PPO"}, {"env_step": 491520, "rew": 28.5064, "rew_std": 20.959, "Agent": "PPO"}, {"env_step": 522240, "rew": 28.5923, "rew_std": 29.5259, "Agent": "PPO"}, {"env_step": 552960, "rew": 23.8657, "rew_std": 22.2134, "Agent": "PPO"}, {"env_step": 583680, "rew": 34.3898, "rew_std": 34.7257, "Agent": "PPO"}, {"env_step": 614400, "rew": 31.3354, "rew_std": 27.7352, "Agent": "PPO"}, {"env_step": 645120, "rew": 29.9966, "rew_std": 23.1932, "Agent": "PPO"}, {"env_step": 675840, "rew": 34.0979, "rew_std": 26.5224, "Agent": "PPO"}, {"env_step": 706560, "rew": 36.3625, "rew_std": 16.5982, "Agent": "PPO"}, {"env_step": 737280, "rew": 37.9557, "rew_std": 17.0515, "Agent": "PPO"}, {"env_step": 768000, "rew": 35.7174, "rew_std": 17.8115, "Agent": "PPO"}, {"env_step": 798720, "rew": 47.5277, "rew_std": 38.7629, "Agent": "PPO"}, {"env_step": 829440, "rew": 44.0949, "rew_std": 38.5072, "Agent": "PPO"}, {"env_step": 860160, "rew": 63.287, "rew_std": 93.1252, "Agent": "PPO"}, {"env_step": 890880, "rew": 62.5291, "rew_std": 96.7548, "Agent": "PPO"}, {"env_step": 921600, "rew": 61.587, "rew_std": 99.9543, "Agent": "PPO"}, {"env_step": 952320, "rew": 66.6847, "rew_std": 99.1066, "Agent": "PPO"}, {"env_step": 983040, "rew": 66.1222, "rew_std": 99.6237, "Agent": "PPO"}, {"env_step": 1013760, "rew": 64.5596, "rew_std": 96.657, "Agent": "PPO"}, {"env_step": 1044480, "rew": 63.9429, "rew_std": 100.59, "Agent": "PPO"}, {"env_step": 1075200, "rew": 67.3104, "rew_std": 99.7823, "Agent": "PPO"}, {"env_step": 1105920, "rew": 68.3072, "rew_std": 99.4767, "Agent": "PPO"}, {"env_step": 1136640, "rew": 69.532, "rew_std": 98.3712, "Agent": "PPO"}, {"env_step": 1167360, "rew": 71.7715, "rew_std": 98.3249, "Agent": "PPO"}, {"env_step": 1198080, "rew": 70.3806, "rew_std": 98.6928, "Agent": "PPO"}, {"env_step": 1228800, "rew": 70.3897, "rew_std": 98.3615, "Agent": "PPO"}, {"env_step": 1259520, "rew": 71.1511, "rew_std": 98.1166, "Agent": "PPO"}, {"env_step": 1290240, "rew": 72.489, "rew_std": 98.3858, "Agent": "PPO"}, {"env_step": 1320960, "rew": 71.0808, "rew_std": 98.8914, "Agent": "PPO"}, {"env_step": 1351680, "rew": 69.7793, "rew_std": 98.9076, "Agent": "PPO"}, {"env_step": 1382400, "rew": 72.0401, "rew_std": 98.174, "Agent": "PPO"}, {"env_step": 1413120, "rew": 72.0039, "rew_std": 98.6329, "Agent": "PPO"}, {"env_step": 1443840, "rew": 71.0677, "rew_std": 99.5856, "Agent": "PPO"}, {"env_step": 1474560, "rew": 73.6586, "rew_std": 97.4667, "Agent": "PPO"}, {"env_step": 1505280, "rew": 70.4729, "rew_std": 99.3265, "Agent": "PPO"}, {"env_step": 1536000, "rew": 72.6155, "rew_std": 98.2909, "Agent": "PPO"}, {"env_step": 1566720, "rew": 75.7714, "rew_std": 97.3342, "Agent": "PPO"}, {"env_step": 1597440, "rew": 73.6515, "rew_std": 97.8379, "Agent": "PPO"}, {"env_step": 1628160, "rew": 73.5543, "rew_std": 98.5735, "Agent": "PPO"}, {"env_step": 1658880, "rew": 70.3039, "rew_std": 98.8956, "Agent": "PPO"}, {"env_step": 1689600, "rew": 75.3847, "rew_std": 97.7091, "Agent": "PPO"}, {"env_step": 1720320, "rew": 70.2479, "rew_std": 99.3575, "Agent": "PPO"}, {"env_step": 1751040, "rew": 75.8085, "rew_std": 97.4939, "Agent": "PPO"}, {"env_step": 1781760, "rew": 78.0713, "rew_std": 96.2682, "Agent": "PPO"}, {"env_step": 1812480, "rew": 70.7668, "rew_std": 99.9198, "Agent": "PPO"}, {"env_step": 1843200, "rew": 74.9315, "rew_std": 98.0385, "Agent": "PPO"}, {"env_step": 1873920, "rew": 74.4566, "rew_std": 97.8105, "Agent": "PPO"}, {"env_step": 1904640, "rew": 72.7717, "rew_std": 99.2967, "Agent": "PPO"}, {"env_step": 1935360, "rew": 76.5677, "rew_std": 96.9867, "Agent": "PPO"}, {"env_step": 1966080, "rew": 78.3815, "rew_std": 96.275, "Agent": "PPO"}, {"env_step": 1996800, "rew": 72.2778, "rew_std": 98.9372, "Agent": "PPO"}, {"env_step": 2027520, "rew": 73.4618, "rew_std": 98.2254, "Agent": "PPO"}, {"env_step": 2058240, "rew": 73.0458, "rew_std": 98.8491, "Agent": "PPO"}, {"env_step": 2088960, "rew": 76.1767, "rew_std": 97.7641, "Agent": "PPO"}, {"env_step": 2119680, "rew": 78.726, "rew_std": 96.0722, "Agent": "PPO"}, {"env_step": 2150400, "rew": 76.7965, "rew_std": 97.3736, "Agent": "PPO"}, {"env_step": 2181120, "rew": 75.239, "rew_std": 98.4263, "Agent": "PPO"}, {"env_step": 2211840, "rew": 76.4495, "rew_std": 97.35, "Agent": "PPO"}, {"env_step": 2242560, "rew": 77.0431, "rew_std": 96.7973, "Agent": "PPO"}, {"env_step": 2273280, "rew": 76.0307, "rew_std": 97.4001, "Agent": "PPO"}, {"env_step": 2304000, "rew": 77.3101, "rew_std": 97.2428, "Agent": "PPO"}, {"env_step": 2334720, "rew": 77.2666, "rew_std": 97.1735, "Agent": "PPO"}, {"env_step": 2365440, "rew": 75.6714, "rew_std": 97.8766, "Agent": "PPO"}, {"env_step": 2396160, "rew": 74.9936, "rew_std": 97.9828, "Agent": "PPO"}, {"env_step": 2426880, "rew": 78.1617, "rew_std": 96.8809, "Agent": "PPO"}, {"env_step": 2457600, "rew": 76.8037, "rew_std": 97.0951, "Agent": "PPO"}, {"env_step": 2488320, "rew": 79.3963, "rew_std": 96.5996, "Agent": "PPO"}, {"env_step": 2519040, "rew": 73.241, "rew_std": 99.1238, "Agent": "PPO"}, {"env_step": 2549760, "rew": 77.9169, "rew_std": 97.3104, "Agent": "PPO"}, {"env_step": 2580480, "rew": 79.9991, "rew_std": 96.0853, "Agent": "PPO"}, {"env_step": 2611200, "rew": 79.6211, "rew_std": 96.5515, "Agent": "PPO"}, {"env_step": 2641920, "rew": 78.5614, "rew_std": 96.9572, "Agent": "PPO"}, {"env_step": 2672640, "rew": 76.6251, "rew_std": 97.7851, "Agent": "PPO"}, {"env_step": 2703360, "rew": 73.7308, "rew_std": 98.7249, "Agent": "PPO"}, {"env_step": 2734080, "rew": 78.168, "rew_std": 96.7511, "Agent": "PPO"}, {"env_step": 2764800, "rew": 78.9345, "rew_std": 96.4013, "Agent": "PPO"}, {"env_step": 2795520, "rew": 77.202, "rew_std": 97.8047, "Agent": "PPO"}, {"env_step": 2826240, "rew": 78.4772, "rew_std": 96.6058, "Agent": "PPO"}, {"env_step": 2856960, "rew": 74.9028, "rew_std": 98.5857, "Agent": "PPO"}, {"env_step": 2887680, "rew": 78.9153, "rew_std": 96.8546, "Agent": "PPO"}, {"env_step": 2918400, "rew": 80.0785, "rew_std": 96.3598, "Agent": "PPO"}, {"env_step": 2949120, "rew": 78.4847, "rew_std": 96.8831, "Agent": "PPO"}, {"env_step": 2979840, "rew": 81.3662, "rew_std": 96.0461, "Agent": "PPO"}, {"env_step": 3010560, "rew": 78.3437, "rew_std": 97.3812, "Agent": "PPO"}, {"env_step": 3041280, "rew": 79.2758, "rew_std": 96.4586, "Agent": "PPO"}, {"env_step": 3072000, "rew": 77.5265, "rew_std": 97.1351, "Agent": "PPO"}, {"env_step": 0, "rew": 0.9618, "rew_std": 3.5215, "Agent": "REINFORCE"}, {"env_step": 30720, "rew": 21.6497, "rew_std": 2.0583, "Agent": "REINFORCE"}, {"env_step": 61440, "rew": 18.8407, "rew_std": 1.6384, "Agent": "REINFORCE"}, {"env_step": 92160, "rew": 21.3462, "rew_std": 1.0074, "Agent": "REINFORCE"}, {"env_step": 122880, "rew": 20.6712, "rew_std": 1.6014, "Agent": "REINFORCE"}, {"env_step": 153600, "rew": 19.9964, "rew_std": 2.1868, "Agent": "REINFORCE"}, {"env_step": 184320, "rew": 22.6191, "rew_std": 4.6673, "Agent": "REINFORCE"}, {"env_step": 215040, "rew": 20.3581, "rew_std": 1.9549, "Agent": "REINFORCE"}, {"env_step": 245760, "rew": 19.8743, "rew_std": 3.3722, "Agent": "REINFORCE"}, {"env_step": 276480, "rew": 21.3424, "rew_std": 2.5073, "Agent": "REINFORCE"}, {"env_step": 307200, "rew": 20.9955, "rew_std": 2.2787, "Agent": "REINFORCE"}, {"env_step": 337920, "rew": 20.4953, "rew_std": 2.7613, "Agent": "REINFORCE"}, {"env_step": 368640, "rew": 22.0264, "rew_std": 4.6766, "Agent": "REINFORCE"}, {"env_step": 399360, "rew": 22.3219, "rew_std": 5.2812, "Agent": "REINFORCE"}, {"env_step": 430080, "rew": 24.307, "rew_std": 6.9595, "Agent": "REINFORCE"}, {"env_step": 460800, "rew": 23.2968, "rew_std": 6.6963, "Agent": "REINFORCE"}, {"env_step": 491520, "rew": 20.2182, "rew_std": 6.8104, "Agent": "REINFORCE"}, {"env_step": 522240, "rew": 24.1779, "rew_std": 6.203, "Agent": "REINFORCE"}, {"env_step": 552960, "rew": 24.7635, "rew_std": 5.5496, "Agent": "REINFORCE"}, {"env_step": 583680, "rew": 25.7685, "rew_std": 7.252, "Agent": "REINFORCE"}, {"env_step": 614400, "rew": 23.2547, "rew_std": 8.9279, "Agent": "REINFORCE"}, {"env_step": 645120, "rew": 25.7667, "rew_std": 8.5028, "Agent": "REINFORCE"}, {"env_step": 675840, "rew": 26.3615, "rew_std": 5.6484, "Agent": "REINFORCE"}, {"env_step": 706560, "rew": 29.3509, "rew_std": 4.2497, "Agent": "REINFORCE"}, {"env_step": 737280, "rew": 27.4427, "rew_std": 4.5532, "Agent": "REINFORCE"}, {"env_step": 768000, "rew": 29.4254, "rew_std": 4.4139, "Agent": "REINFORCE"}, {"env_step": 798720, "rew": 28.885, "rew_std": 4.5786, "Agent": "REINFORCE"}, {"env_step": 829440, "rew": 29.5476, "rew_std": 5.0349, "Agent": "REINFORCE"}, {"env_step": 860160, "rew": 29.6066, "rew_std": 5.663, "Agent": "REINFORCE"}, {"env_step": 890880, "rew": 29.6368, "rew_std": 5.2243, "Agent": "REINFORCE"}, {"env_step": 921600, "rew": 29.8861, "rew_std": 5.0542, "Agent": "REINFORCE"}, {"env_step": 952320, "rew": 29.8157, "rew_std": 5.0216, "Agent": "REINFORCE"}, {"env_step": 983040, "rew": 30.0077, "rew_std": 7.2744, "Agent": "REINFORCE"}, {"env_step": 1013760, "rew": 30.5773, "rew_std": 6.8568, "Agent": "REINFORCE"}, {"env_step": 1044480, "rew": 29.8747, "rew_std": 6.5815, "Agent": "REINFORCE"}, {"env_step": 1075200, "rew": 28.6939, "rew_std": 8.3452, "Agent": "REINFORCE"}, {"env_step": 1105920, "rew": 30.8997, "rew_std": 4.6416, "Agent": "REINFORCE"}, {"env_step": 1136640, "rew": 32.1676, "rew_std": 3.6635, "Agent": "REINFORCE"}, {"env_step": 1167360, "rew": 33.282, "rew_std": 3.9632, "Agent": "REINFORCE"}, {"env_step": 1198080, "rew": 32.7282, "rew_std": 4.2844, "Agent": "REINFORCE"}, {"env_step": 1228800, "rew": 32.175, "rew_std": 1.4538, "Agent": "REINFORCE"}, {"env_step": 1259520, "rew": 32.3137, "rew_std": 2.4142, "Agent": "REINFORCE"}, {"env_step": 1290240, "rew": 33.0787, "rew_std": 1.7409, "Agent": "REINFORCE"}, {"env_step": 1320960, "rew": 32.4292, "rew_std": 2.7709, "Agent": "REINFORCE"}, {"env_step": 1351680, "rew": 33.5495, "rew_std": 1.8482, "Agent": "REINFORCE"}, {"env_step": 1382400, "rew": 33.2865, "rew_std": 1.8124, "Agent": "REINFORCE"}, {"env_step": 1413120, "rew": 34.02, "rew_std": 1.5783, "Agent": "REINFORCE"}, {"env_step": 1443840, "rew": 32.4269, "rew_std": 1.3609, "Agent": "REINFORCE"}, {"env_step": 1474560, "rew": 35.2871, "rew_std": 1.4372, "Agent": "REINFORCE"}, {"env_step": 1505280, "rew": 32.8925, "rew_std": 2.0544, "Agent": "REINFORCE"}, {"env_step": 1536000, "rew": 33.947, "rew_std": 2.344, "Agent": "REINFORCE"}, {"env_step": 1566720, "rew": 33.2722, "rew_std": 2.4675, "Agent": "REINFORCE"}, {"env_step": 1597440, "rew": 33.1716, "rew_std": 2.4712, "Agent": "REINFORCE"}, {"env_step": 1628160, "rew": 31.8048, "rew_std": 3.1568, "Agent": "REINFORCE"}, {"env_step": 1658880, "rew": 32.1113, "rew_std": 3.3193, "Agent": "REINFORCE"}, {"env_step": 1689600, "rew": 32.7537, "rew_std": 2.7257, "Agent": "REINFORCE"}, {"env_step": 1720320, "rew": 32.2556, "rew_std": 2.9733, "Agent": "REINFORCE"}, {"env_step": 1751040, "rew": 34.5128, "rew_std": 2.3075, "Agent": "REINFORCE"}, {"env_step": 1781760, "rew": 34.341, "rew_std": 2.7978, "Agent": "REINFORCE"}, {"env_step": 1812480, "rew": 32.339, "rew_std": 2.7514, "Agent": "REINFORCE"}, {"env_step": 1843200, "rew": 34.0286, "rew_std": 2.9285, "Agent": "REINFORCE"}, {"env_step": 1873920, "rew": 32.9519, "rew_std": 2.0919, "Agent": "REINFORCE"}, {"env_step": 1904640, "rew": 33.9109, "rew_std": 2.8357, "Agent": "REINFORCE"}, {"env_step": 1935360, "rew": 34.4966, "rew_std": 1.0048, "Agent": "REINFORCE"}, {"env_step": 1966080, "rew": 34.5729, "rew_std": 1.5995, "Agent": "REINFORCE"}, {"env_step": 1996800, "rew": 33.9055, "rew_std": 1.9507, "Agent": "REINFORCE"}, {"env_step": 2027520, "rew": 33.6248, "rew_std": 2.1721, "Agent": "REINFORCE"}, {"env_step": 2058240, "rew": 33.5189, "rew_std": 2.9632, "Agent": "REINFORCE"}, {"env_step": 2088960, "rew": 33.0232, "rew_std": 3.8123, "Agent": "REINFORCE"}, {"env_step": 2119680, "rew": 33.4146, "rew_std": 3.783, "Agent": "REINFORCE"}, {"env_step": 2150400, "rew": 32.7739, "rew_std": 3.8725, "Agent": "REINFORCE"}, {"env_step": 2181120, "rew": 32.1464, "rew_std": 3.4452, "Agent": "REINFORCE"}, {"env_step": 2211840, "rew": 32.4067, "rew_std": 3.6292, "Agent": "REINFORCE"}, {"env_step": 2242560, "rew": 32.8531, "rew_std": 4.2917, "Agent": "REINFORCE"}, {"env_step": 2273280, "rew": 31.3723, "rew_std": 4.6388, "Agent": "REINFORCE"}, {"env_step": 2304000, "rew": 31.7266, "rew_std": 4.3836, "Agent": "REINFORCE"}, {"env_step": 2334720, "rew": 31.9897, "rew_std": 4.348, "Agent": "REINFORCE"}, {"env_step": 2365440, "rew": 33.2492, "rew_std": 4.0503, "Agent": "REINFORCE"}, {"env_step": 2396160, "rew": 32.555, "rew_std": 3.7809, "Agent": "REINFORCE"}, {"env_step": 2426880, "rew": 32.5059, "rew_std": 3.2845, "Agent": "REINFORCE"}, {"env_step": 2457600, "rew": 31.1916, "rew_std": 4.5898, "Agent": "REINFORCE"}, {"env_step": 2488320, "rew": 32.4978, "rew_std": 3.9197, "Agent": "REINFORCE"}, {"env_step": 2519040, "rew": 29.8591, "rew_std": 6.5104, "Agent": "REINFORCE"}, {"env_step": 2549760, "rew": 31.2884, "rew_std": 4.2571, "Agent": "REINFORCE"}, {"env_step": 2580480, "rew": 32.3156, "rew_std": 4.9718, "Agent": "REINFORCE"}, {"env_step": 2611200, "rew": 32.6244, "rew_std": 5.1626, "Agent": "REINFORCE"}, {"env_step": 2641920, "rew": 31.9868, "rew_std": 5.7592, "Agent": "REINFORCE"}, {"env_step": 2672640, "rew": 32.893, "rew_std": 3.5311, "Agent": "REINFORCE"}, {"env_step": 2703360, "rew": 32.205, "rew_std": 4.235, "Agent": "REINFORCE"}, {"env_step": 2734080, "rew": 32.1189, "rew_std": 4.6476, "Agent": "REINFORCE"}, {"env_step": 2764800, "rew": 32.3047, "rew_std": 4.5396, "Agent": "REINFORCE"}, {"env_step": 2795520, "rew": 32.6739, "rew_std": 4.9249, "Agent": "REINFORCE"}, {"env_step": 2826240, "rew": 33.2044, "rew_std": 4.504, "Agent": "REINFORCE"}, {"env_step": 2856960, "rew": 33.0202, "rew_std": 3.5788, "Agent": "REINFORCE"}, {"env_step": 2887680, "rew": 33.5991, "rew_std": 3.9779, "Agent": "REINFORCE"}, {"env_step": 2918400, "rew": 33.6818, "rew_std": 3.3053, "Agent": "REINFORCE"}, {"env_step": 2949120, "rew": 33.2439, "rew_std": 3.7153, "Agent": "REINFORCE"}, {"env_step": 2979840, "rew": 35.0602, "rew_std": 2.1509, "Agent": "REINFORCE"}, {"env_step": 3010560, "rew": 34.144, "rew_std": 1.9498, "Agent": "REINFORCE"}, {"env_step": 3041280, "rew": 34.0212, "rew_std": 2.8477, "Agent": "REINFORCE"}, {"env_step": 3072000, "rew": 33.22, "rew_std": 2.5058, "Agent": "REINFORCE"}, {"env_step": 3102720, "rew": 34.3311, "rew_std": 2.0098, "Agent": "REINFORCE"}, {"env_step": 3133440, "rew": 34.5845, "rew_std": 2.487, "Agent": "REINFORCE"}, {"env_step": 3164160, "rew": 33.8012, "rew_std": 2.8445, "Agent": "REINFORCE"}, {"env_step": 3194880, "rew": 33.4961, "rew_std": 2.9447, "Agent": "REINFORCE"}, {"env_step": 3225600, "rew": 34.1328, "rew_std": 2.1202, "Agent": "REINFORCE"}, {"env_step": 3256320, "rew": 33.9942, "rew_std": 2.1889, "Agent": "REINFORCE"}, {"env_step": 3287040, "rew": 34.4959, "rew_std": 3.1445, "Agent": "REINFORCE"}, {"env_step": 3317760, "rew": 34.1392, "rew_std": 4.2095, "Agent": "REINFORCE"}, {"env_step": 3348480, "rew": 34.3033, "rew_std": 4.2899, "Agent": "REINFORCE"}, {"env_step": 3379200, "rew": 34.2744, "rew_std": 3.3235, "Agent": "REINFORCE"}, {"env_step": 3409920, "rew": 32.5941, "rew_std": 5.7054, "Agent": "REINFORCE"}, {"env_step": 3440640, "rew": 32.5893, "rew_std": 5.6679, "Agent": "REINFORCE"}, {"env_step": 3471360, "rew": 32.8065, "rew_std": 4.9803, "Agent": "REINFORCE"}, {"env_step": 3502080, "rew": 34.058, "rew_std": 4.703, "Agent": "REINFORCE"}, {"env_step": 3532800, "rew": 32.7505, "rew_std": 6.0165, "Agent": "REINFORCE"}, {"env_step": 3563520, "rew": 32.9924, "rew_std": 5.2966, "Agent": "REINFORCE"}, {"env_step": 3594240, "rew": 33.5606, "rew_std": 5.081, "Agent": "REINFORCE"}, {"env_step": 3624960, "rew": 34.3386, "rew_std": 4.0346, "Agent": "REINFORCE"}, {"env_step": 3655680, "rew": 33.9057, "rew_std": 4.381, "Agent": "REINFORCE"}, {"env_step": 3686400, "rew": 34.9121, "rew_std": 2.9535, "Agent": "REINFORCE"}, {"env_step": 3717120, "rew": 33.749, "rew_std": 3.1327, "Agent": "REINFORCE"}, {"env_step": 3747840, "rew": 34.2743, "rew_std": 3.6415, "Agent": "REINFORCE"}, {"env_step": 3778560, "rew": 35.3061, "rew_std": 2.1058, "Agent": "REINFORCE"}, {"env_step": 3809280, "rew": 34.3247, "rew_std": 2.3545, "Agent": "REINFORCE"}, {"env_step": 3840000, "rew": 34.8757, "rew_std": 3.2262, "Agent": "REINFORCE"}, {"env_step": 3870720, "rew": 34.947, "rew_std": 2.9427, "Agent": "REINFORCE"}, {"env_step": 3901440, "rew": 34.7932, "rew_std": 2.9123, "Agent": "REINFORCE"}, {"env_step": 3932160, "rew": 34.0642, "rew_std": 5.3443, "Agent": "REINFORCE"}, {"env_step": 3962880, "rew": 34.2344, "rew_std": 4.4914, "Agent": "REINFORCE"}, {"env_step": 3993600, "rew": 33.9348, "rew_std": 4.2284, "Agent": "REINFORCE"}, {"env_step": 4024320, "rew": 33.3052, "rew_std": 4.1758, "Agent": "REINFORCE"}, {"env_step": 4055040, "rew": 33.9265, "rew_std": 5.0275, "Agent": "REINFORCE"}, {"env_step": 4085760, "rew": 32.6343, "rew_std": 4.9864, "Agent": "REINFORCE"}, {"env_step": 4116480, "rew": 33.7496, "rew_std": 3.5404, "Agent": "REINFORCE"}, {"env_step": 4147200, "rew": 33.345, "rew_std": 4.1666, "Agent": "REINFORCE"}, {"env_step": 4177920, "rew": 33.9686, "rew_std": 2.8585, "Agent": "REINFORCE"}, {"env_step": 4208640, "rew": 34.0236, "rew_std": 3.1606, "Agent": "REINFORCE"}, {"env_step": 4239360, "rew": 34.0668, "rew_std": 3.1711, "Agent": "REINFORCE"}, {"env_step": 4270080, "rew": 34.0769, "rew_std": 2.6831, "Agent": "REINFORCE"}, {"env_step": 4300800, "rew": 33.2015, "rew_std": 3.4703, "Agent": "REINFORCE"}, {"env_step": 4331520, "rew": 33.1703, "rew_std": 3.6308, "Agent": "REINFORCE"}, {"env_step": 4362240, "rew": 33.7326, "rew_std": 3.5252, "Agent": "REINFORCE"}, {"env_step": 4392960, "rew": 33.1725, "rew_std": 3.596, "Agent": "REINFORCE"}, {"env_step": 4423680, "rew": 32.7075, "rew_std": 4.6077, "Agent": "REINFORCE"}, {"env_step": 4454400, "rew": 32.9265, "rew_std": 3.6788, "Agent": "REINFORCE"}, {"env_step": 4485120, "rew": 33.281, "rew_std": 3.5235, "Agent": "REINFORCE"}, {"env_step": 4515840, "rew": 32.0004, "rew_std": 4.0413, "Agent": "REINFORCE"}, {"env_step": 4546560, "rew": 32.252, "rew_std": 3.6599, "Agent": "REINFORCE"}, {"env_step": 4577280, "rew": 33.465, "rew_std": 4.4369, "Agent": "REINFORCE"}, {"env_step": 4608000, "rew": 33.002, "rew_std": 4.3731, "Agent": "REINFORCE"}, {"env_step": 4638720, "rew": 34.0664, "rew_std": 4.0684, "Agent": "REINFORCE"}, {"env_step": 4669440, "rew": 34.2007, "rew_std": 4.326, "Agent": "REINFORCE"}, {"env_step": 4700160, "rew": 33.1797, "rew_std": 3.901, "Agent": "REINFORCE"}, {"env_step": 4730880, "rew": 33.247, "rew_std": 3.4091, "Agent": "REINFORCE"}, {"env_step": 4761600, "rew": 33.1553, "rew_std": 4.0657, "Agent": "REINFORCE"}, {"env_step": 4792320, "rew": 32.6889, "rew_std": 4.1791, "Agent": "REINFORCE"}, {"env_step": 4823040, "rew": 33.854, "rew_std": 3.5738, "Agent": "REINFORCE"}, {"env_step": 4853760, "rew": 33.5175, "rew_std": 3.3581, "Agent": "REINFORCE"}, {"env_step": 4884480, "rew": 32.3676, "rew_std": 2.9989, "Agent": "REINFORCE"}, {"env_step": 4915200, "rew": 32.6538, "rew_std": 3.5436, "Agent": "REINFORCE"}, {"env_step": 4945920, "rew": 31.9695, "rew_std": 4.0538, "Agent": "REINFORCE"}, {"env_step": 4976640, "rew": 33.8209, "rew_std": 3.4171, "Agent": "REINFORCE"}, {"env_step": 5007360, "rew": 33.2326, "rew_std": 4.5398, "Agent": "REINFORCE"}, {"env_step": 5038080, "rew": 33.0102, "rew_std": 5.1255, "Agent": "REINFORCE"}, {"env_step": 5068800, "rew": 32.7645, "rew_std": 3.9751, "Agent": "REINFORCE"}, {"env_step": 5099520, "rew": 33.3095, "rew_std": 3.5838, "Agent": "REINFORCE"}, {"env_step": 5130240, "rew": 33.9164, "rew_std": 4.5219, "Agent": "REINFORCE"}, {"env_step": 5160960, "rew": 33.2565, "rew_std": 3.1545, "Agent": "REINFORCE"}, {"env_step": 5191680, "rew": 33.6445, "rew_std": 3.4789, "Agent": "REINFORCE"}, {"env_step": 5222400, "rew": 32.9714, "rew_std": 5.0788, "Agent": "REINFORCE"}, {"env_step": 5253120, "rew": 33.2731, "rew_std": 4.1268, "Agent": "REINFORCE"}, {"env_step": 5283840, "rew": 34.2387, "rew_std": 3.831, "Agent": "REINFORCE"}, {"env_step": 5314560, "rew": 33.0195, "rew_std": 4.4602, "Agent": "REINFORCE"}, {"env_step": 5345280, "rew": 33.0358, "rew_std": 4.7634, "Agent": "REINFORCE"}, {"env_step": 5376000, "rew": 33.066, "rew_std": 5.381, "Agent": "REINFORCE"}, {"env_step": 5406720, "rew": 32.8663, "rew_std": 4.5141, "Agent": "REINFORCE"}, {"env_step": 5437440, "rew": 32.182, "rew_std": 5.3874, "Agent": "REINFORCE"}, {"env_step": 5468160, "rew": 32.1664, "rew_std": 6.5647, "Agent": "REINFORCE"}, {"env_step": 5498880, "rew": 32.4971, "rew_std": 5.5007, "Agent": "REINFORCE"}, {"env_step": 5529600, "rew": 31.575, "rew_std": 5.0883, "Agent": "REINFORCE"}, {"env_step": 5560320, "rew": 33.3368, "rew_std": 3.9527, "Agent": "REINFORCE"}, {"env_step": 5591040, "rew": 33.6293, "rew_std": 4.5512, "Agent": "REINFORCE"}, {"env_step": 5621760, "rew": 33.625, "rew_std": 3.9672, "Agent": "REINFORCE"}, {"env_step": 5652480, "rew": 33.2956, "rew_std": 4.5379, "Agent": "REINFORCE"}, {"env_step": 5683200, "rew": 33.2566, "rew_std": 4.1496, "Agent": "REINFORCE"}, {"env_step": 5713920, "rew": 33.7133, "rew_std": 4.1502, "Agent": "REINFORCE"}, {"env_step": 5744640, "rew": 34.8932, "rew_std": 3.706, "Agent": "REINFORCE"}, {"env_step": 5775360, "rew": 34.6708, "rew_std": 3.8852, "Agent": "REINFORCE"}, {"env_step": 5806080, "rew": 34.7482, "rew_std": 3.4388, "Agent": "REINFORCE"}, {"env_step": 5836800, "rew": 34.3264, "rew_std": 3.6939, "Agent": "REINFORCE"}, {"env_step": 5867520, "rew": 33.9703, "rew_std": 3.2207, "Agent": "REINFORCE"}, {"env_step": 5898240, "rew": 34.8393, "rew_std": 4.2321, "Agent": "REINFORCE"}, {"env_step": 5928960, "rew": 34.9294, "rew_std": 4.0082, "Agent": "REINFORCE"}, {"env_step": 5959680, "rew": 34.1719, "rew_std": 4.7017, "Agent": "REINFORCE"}, {"env_step": 5990400, "rew": 33.3955, "rew_std": 4.192, "Agent": "REINFORCE"}, {"env_step": 6021120, "rew": 32.3953, "rew_std": 5.2512, "Agent": "REINFORCE"}, {"env_step": 6051840, "rew": 33.3806, "rew_std": 4.3844, "Agent": "REINFORCE"}, {"env_step": 6082560, "rew": 33.129, "rew_std": 5.484, "Agent": "REINFORCE"}, {"env_step": 6113280, "rew": 32.5978, "rew_std": 5.0187, "Agent": "REINFORCE"}, {"env_step": 6144000, "rew": 33.2739, "rew_std": 3.9319, "Agent": "REINFORCE"}, {"env_step": 6174720, "rew": 32.561, "rew_std": 4.5382, "Agent": "REINFORCE"}, {"env_step": 6205440, "rew": 32.4747, "rew_std": 5.3743, "Agent": "REINFORCE"}, {"env_step": 6236160, "rew": 32.4632, "rew_std": 5.8385, "Agent": "REINFORCE"}, {"env_step": 6266880, "rew": 32.6665, "rew_std": 5.6886, "Agent": "REINFORCE"}, {"env_step": 6297600, "rew": 31.5359, "rew_std": 6.0376, "Agent": "REINFORCE"}, {"env_step": 6328320, "rew": 32.0438, "rew_std": 5.7212, "Agent": "REINFORCE"}, {"env_step": 6359040, "rew": 30.95, "rew_std": 7.1644, "Agent": "REINFORCE"}, {"env_step": 6389760, "rew": 32.0496, "rew_std": 6.4098, "Agent": "REINFORCE"}, {"env_step": 6420480, "rew": 32.5061, "rew_std": 5.9136, "Agent": "REINFORCE"}, {"env_step": 6451200, "rew": 32.4526, "rew_std": 5.9224, "Agent": "REINFORCE"}, {"env_step": 6481920, "rew": 32.3688, "rew_std": 6.1382, "Agent": "REINFORCE"}, {"env_step": 6512640, "rew": 32.2328, "rew_std": 4.6938, "Agent": "REINFORCE"}, {"env_step": 6543360, "rew": 31.9146, "rew_std": 4.3941, "Agent": "REINFORCE"}, {"env_step": 6574080, "rew": 31.4479, "rew_std": 6.3545, "Agent": "REINFORCE"}, {"env_step": 6604800, "rew": 30.8629, "rew_std": 5.4676, "Agent": "REINFORCE"}, {"env_step": 6635520, "rew": 30.7007, "rew_std": 5.5163, "Agent": "REINFORCE"}, {"env_step": 6666240, "rew": 31.7723, "rew_std": 5.9939, "Agent": "REINFORCE"}, {"env_step": 6696960, "rew": 32.615, "rew_std": 5.8788, "Agent": "REINFORCE"}, {"env_step": 6727680, "rew": 31.6344, "rew_std": 4.9203, "Agent": "REINFORCE"}, {"env_step": 6758400, "rew": 32.0157, "rew_std": 5.063, "Agent": "REINFORCE"}, {"env_step": 6789120, "rew": 31.7169, "rew_std": 4.5693, "Agent": "REINFORCE"}, {"env_step": 6819840, "rew": 32.4026, "rew_std": 4.4906, "Agent": "REINFORCE"}, {"env_step": 6850560, "rew": 30.6853, "rew_std": 5.5575, "Agent": "REINFORCE"}, {"env_step": 6881280, "rew": 31.1717, "rew_std": 5.9472, "Agent": "REINFORCE"}, {"env_step": 6912000, "rew": 31.6307, "rew_std": 4.759, "Agent": "REINFORCE"}, {"env_step": 6942720, "rew": 31.6028, "rew_std": 5.7942, "Agent": "REINFORCE"}, {"env_step": 6973440, "rew": 31.8346, "rew_std": 5.6822, "Agent": "REINFORCE"}, {"env_step": 7004160, "rew": 31.9882, "rew_std": 5.955, "Agent": "REINFORCE"}, {"env_step": 7034880, "rew": 32.382, "rew_std": 5.8684, "Agent": "REINFORCE"}, {"env_step": 7065600, "rew": 31.0684, "rew_std": 6.2851, "Agent": "REINFORCE"}, {"env_step": 7096320, "rew": 30.9765, "rew_std": 5.6595, "Agent": "REINFORCE"}, {"env_step": 7127040, "rew": 31.9173, "rew_std": 5.1753, "Agent": "REINFORCE"}, {"env_step": 7157760, "rew": 31.3002, "rew_std": 5.7539, "Agent": "REINFORCE"}, {"env_step": 7188480, "rew": 32.7446, "rew_std": 5.5887, "Agent": "REINFORCE"}, {"env_step": 7219200, "rew": 31.3714, "rew_std": 6.5631, "Agent": "REINFORCE"}, {"env_step": 7249920, "rew": 32.337, "rew_std": 5.475, "Agent": "REINFORCE"}, {"env_step": 7280640, "rew": 31.2519, "rew_std": 5.9219, "Agent": "REINFORCE"}, {"env_step": 7311360, "rew": 33.2408, "rew_std": 4.933, "Agent": "REINFORCE"}, {"env_step": 7342080, "rew": 32.6244, "rew_std": 5.3486, "Agent": "REINFORCE"}, {"env_step": 7372800, "rew": 32.2818, "rew_std": 5.1002, "Agent": "REINFORCE"}, {"env_step": 7403520, "rew": 32.4273, "rew_std": 5.6617, "Agent": "REINFORCE"}, {"env_step": 7434240, "rew": 32.5061, "rew_std": 5.56, "Agent": "REINFORCE"}, {"env_step": 7464960, "rew": 31.0894, "rew_std": 5.8324, "Agent": "REINFORCE"}, {"env_step": 7495680, "rew": 31.6037, "rew_std": 6.0186, "Agent": "REINFORCE"}, {"env_step": 7526400, "rew": 32.7055, "rew_std": 6.2364, "Agent": "REINFORCE"}, {"env_step": 7557120, "rew": 32.8096, "rew_std": 4.6631, "Agent": "REINFORCE"}, {"env_step": 7587840, "rew": 32.5216, "rew_std": 5.6356, "Agent": "REINFORCE"}, {"env_step": 7618560, "rew": 32.3616, "rew_std": 6.5821, "Agent": "REINFORCE"}, {"env_step": 7649280, "rew": 32.4397, "rew_std": 5.0747, "Agent": "REINFORCE"}, {"env_step": 7680000, "rew": 32.0721, "rew_std": 6.0841, "Agent": "REINFORCE"}, {"env_step": 7710720, "rew": 32.0599, "rew_std": 5.1005, "Agent": "REINFORCE"}, {"env_step": 7741440, "rew": 32.512, "rew_std": 5.0088, "Agent": "REINFORCE"}, {"env_step": 7772160, "rew": 33.6162, "rew_std": 4.6397, "Agent": "REINFORCE"}, {"env_step": 7802880, "rew": 32.9104, "rew_std": 4.5188, "Agent": "REINFORCE"}, {"env_step": 7833600, "rew": 32.8331, "rew_std": 4.9909, "Agent": "REINFORCE"}, {"env_step": 7864320, "rew": 32.7758, "rew_std": 4.5786, "Agent": "REINFORCE"}, {"env_step": 7895040, "rew": 34.003, "rew_std": 4.3646, "Agent": "REINFORCE"}, {"env_step": 7925760, "rew": 32.4516, "rew_std": 5.5562, "Agent": "REINFORCE"}, {"env_step": 7956480, "rew": 34.1675, "rew_std": 4.9455, "Agent": "REINFORCE"}, {"env_step": 7987200, "rew": 33.6632, "rew_std": 4.0695, "Agent": "REINFORCE"}, {"env_step": 8017920, "rew": 34.0068, "rew_std": 4.107, "Agent": "REINFORCE"}, {"env_step": 8048640, "rew": 33.4871, "rew_std": 4.6873, "Agent": "REINFORCE"}, {"env_step": 8079360, "rew": 33.7246, "rew_std": 4.5231, "Agent": "REINFORCE"}, {"env_step": 8110080, "rew": 32.5878, "rew_std": 5.5223, "Agent": "REINFORCE"}, {"env_step": 8140800, "rew": 33.9746, "rew_std": 4.7386, "Agent": "REINFORCE"}, {"env_step": 8171520, "rew": 32.6613, "rew_std": 5.3638, "Agent": "REINFORCE"}, {"env_step": 8202240, "rew": 33.0588, "rew_std": 5.1075, "Agent": "REINFORCE"}, {"env_step": 8232960, "rew": 33.9021, "rew_std": 5.0514, "Agent": "REINFORCE"}, {"env_step": 8263680, "rew": 34.0947, "rew_std": 4.5952, "Agent": "REINFORCE"}, {"env_step": 8294400, "rew": 32.703, "rew_std": 4.7867, "Agent": "REINFORCE"}, {"env_step": 8325120, "rew": 34.1386, "rew_std": 4.75, "Agent": "REINFORCE"}, {"env_step": 8355840, "rew": 33.9661, "rew_std": 4.4242, "Agent": "REINFORCE"}, {"env_step": 8386560, "rew": 34.7524, "rew_std": 4.6168, "Agent": "REINFORCE"}, {"env_step": 8417280, "rew": 33.9398, "rew_std": 4.263, "Agent": "REINFORCE"}, {"env_step": 8448000, "rew": 33.1136, "rew_std": 5.6609, "Agent": "REINFORCE"}, {"env_step": 8478720, "rew": 34.2965, "rew_std": 4.169, "Agent": "REINFORCE"}, {"env_step": 8509440, "rew": 33.6618, "rew_std": 4.5977, "Agent": "REINFORCE"}, {"env_step": 8540160, "rew": 33.6729, "rew_std": 4.0488, "Agent": "REINFORCE"}, {"env_step": 8570880, "rew": 34.9866, "rew_std": 4.0343, "Agent": "REINFORCE"}, {"env_step": 8601600, "rew": 34.5048, "rew_std": 4.5793, "Agent": "REINFORCE"}, {"env_step": 8632320, "rew": 34.0164, "rew_std": 4.0996, "Agent": "REINFORCE"}, {"env_step": 8663040, "rew": 34.5609, "rew_std": 3.7994, "Agent": "REINFORCE"}, {"env_step": 8693760, "rew": 33.8448, "rew_std": 4.3228, "Agent": "REINFORCE"}, {"env_step": 8724480, "rew": 33.4444, "rew_std": 4.5564, "Agent": "REINFORCE"}, {"env_step": 8755200, "rew": 33.525, "rew_std": 4.4545, "Agent": "REINFORCE"}, {"env_step": 8785920, "rew": 33.7675, "rew_std": 5.0768, "Agent": "REINFORCE"}, {"env_step": 8816640, "rew": 34.8722, "rew_std": 3.4362, "Agent": "REINFORCE"}, {"env_step": 8847360, "rew": 33.9305, "rew_std": 3.8452, "Agent": "REINFORCE"}, {"env_step": 8878080, "rew": 33.381, "rew_std": 4.6669, "Agent": "REINFORCE"}, {"env_step": 8908800, "rew": 34.5151, "rew_std": 4.0725, "Agent": "REINFORCE"}, {"env_step": 8939520, "rew": 34.5288, "rew_std": 4.3255, "Agent": "REINFORCE"}, {"env_step": 8970240, "rew": 34.3572, "rew_std": 4.0005, "Agent": "REINFORCE"}, {"env_step": 9000960, "rew": 35.5993, "rew_std": 4.8346, "Agent": "REINFORCE"}, {"env_step": 9031680, "rew": 35.0188, "rew_std": 3.7941, "Agent": "REINFORCE"}, {"env_step": 9062400, "rew": 32.4251, "rew_std": 4.2903, "Agent": "REINFORCE"}, {"env_step": 9093120, "rew": 34.6576, "rew_std": 4.1041, "Agent": "REINFORCE"}, {"env_step": 9123840, "rew": 33.9335, "rew_std": 4.2551, "Agent": "REINFORCE"}, {"env_step": 9154560, "rew": 33.5112, "rew_std": 4.5228, "Agent": "REINFORCE"}, {"env_step": 9185280, "rew": 35.0373, "rew_std": 4.2203, "Agent": "REINFORCE"}, {"env_step": 9216000, "rew": 33.1811, "rew_std": 4.8138, "Agent": "REINFORCE"}, {"env_step": 9246720, "rew": 35.6357, "rew_std": 2.5664, "Agent": "REINFORCE"}, {"env_step": 9277440, "rew": 35.1787, "rew_std": 3.656, "Agent": "REINFORCE"}, {"env_step": 9308160, "rew": 34.1176, "rew_std": 4.1731, "Agent": "REINFORCE"}, {"env_step": 9338880, "rew": 32.852, "rew_std": 5.0998, "Agent": "REINFORCE"}, {"env_step": 9369600, "rew": 34.1419, "rew_std": 4.3647, "Agent": "REINFORCE"}, {"env_step": 9400320, "rew": 34.9605, "rew_std": 4.4513, "Agent": "REINFORCE"}, {"env_step": 9431040, "rew": 34.0714, "rew_std": 4.1276, "Agent": "REINFORCE"}, {"env_step": 9461760, "rew": 33.9586, "rew_std": 4.0475, "Agent": "REINFORCE"}, {"env_step": 9492480, "rew": 32.7241, "rew_std": 5.6342, "Agent": "REINFORCE"}, {"env_step": 9523200, "rew": 34.8576, "rew_std": 4.3271, "Agent": "REINFORCE"}, {"env_step": 9553920, "rew": 34.5418, "rew_std": 4.3088, "Agent": "REINFORCE"}, {"env_step": 9584640, "rew": 33.4314, "rew_std": 4.8686, "Agent": "REINFORCE"}, {"env_step": 9615360, "rew": 34.31, "rew_std": 4.2917, "Agent": "REINFORCE"}, {"env_step": 9646080, "rew": 34.1281, "rew_std": 4.2791, "Agent": "REINFORCE"}, {"env_step": 9676800, "rew": 34.2341, "rew_std": 4.776, "Agent": "REINFORCE"}, {"env_step": 9707520, "rew": 34.8616, "rew_std": 3.861, "Agent": "REINFORCE"}, {"env_step": 9738240, "rew": 33.6738, "rew_std": 4.3368, "Agent": "REINFORCE"}, {"env_step": 9768960, "rew": 34.363, "rew_std": 4.2437, "Agent": "REINFORCE"}, {"env_step": 9799680, "rew": 33.7762, "rew_std": 4.6374, "Agent": "REINFORCE"}, {"env_step": 9830400, "rew": 34.3446, "rew_std": 4.5658, "Agent": "REINFORCE"}, {"env_step": 9861120, "rew": 34.2667, "rew_std": 4.1647, "Agent": "REINFORCE"}, {"env_step": 9891840, "rew": 34.6621, "rew_std": 4.0091, "Agent": "REINFORCE"}, {"env_step": 9922560, "rew": 33.8437, "rew_std": 4.2047, "Agent": "REINFORCE"}, {"env_step": 9953280, "rew": 33.6302, "rew_std": 4.7983, "Agent": "REINFORCE"}, {"env_step": 9984000, "rew": 34.0698, "rew_std": 4.1396, "Agent": "REINFORCE"}, {"env_step": 10014720, "rew": 32.9844, "rew_std": 5.1013, "Agent": "REINFORCE"}, {"env_step": 10045440, "rew": 33.5285, "rew_std": 4.4086, "Agent": "REINFORCE"}, {"env_step": 10076160, "rew": 34.0825, "rew_std": 4.5864, "Agent": "REINFORCE"}, {"env_step": 10106880, "rew": 34.2197, "rew_std": 4.7614, "Agent": "REINFORCE"}, {"env_step": 10137600, "rew": 33.4028, "rew_std": 4.4683, "Agent": "REINFORCE"}, {"env_step": 10168320, "rew": 33.7714, "rew_std": 5.2796, "Agent": "REINFORCE"}, {"env_step": 10199040, "rew": 33.7844, "rew_std": 5.0687, "Agent": "REINFORCE"}, {"env_step": 10229760, "rew": 34.2833, "rew_std": 4.5147, "Agent": "REINFORCE"}, {"env_step": 0, "rew": 1.0013, "rew_std": 3.8528, "Agent": "A2C"}, {"env_step": 30000, "rew": 29.7993, "rew_std": 3.9754, "Agent": "A2C"}, {"env_step": 60000, "rew": 30.3959, "rew_std": 3.2532, "Agent": "A2C"}, {"env_step": 90000, "rew": 28.3456, "rew_std": 4.0923, "Agent": "A2C"}, {"env_step": 120000, "rew": 27.5501, "rew_std": 6.0253, "Agent": "A2C"}, {"env_step": 150000, "rew": 28.0867, "rew_std": 4.9332, "Agent": "A2C"}, {"env_step": 180000, "rew": 28.6474, "rew_std": 4.651, "Agent": "A2C"}, {"env_step": 210000, "rew": 30.9932, "rew_std": 4.2765, "Agent": "A2C"}, {"env_step": 240000, "rew": 28.4719, "rew_std": 5.4846, "Agent": "A2C"}, {"env_step": 270000, "rew": 31.8838, "rew_std": 5.2627, "Agent": "A2C"}, {"env_step": 300000, "rew": 30.2396, "rew_std": 5.2384, "Agent": "A2C"}, {"env_step": 330000, "rew": 30.2113, "rew_std": 4.9179, "Agent": "A2C"}, {"env_step": 360000, "rew": 30.1494, "rew_std": 4.1007, "Agent": "A2C"}, {"env_step": 390000, "rew": 30.4588, "rew_std": 5.2267, "Agent": "A2C"}, {"env_step": 420000, "rew": 26.5, "rew_std": 11.502, "Agent": "A2C"}, {"env_step": 450000, "rew": 25.217, "rew_std": 13.1053, "Agent": "A2C"}, {"env_step": 480000, "rew": 29.6318, "rew_std": 3.5898, "Agent": "A2C"}, {"env_step": 510000, "rew": 31.3575, "rew_std": 3.5037, "Agent": "A2C"}, {"env_step": 540000, "rew": 29.2128, "rew_std": 7.123, "Agent": "A2C"}, {"env_step": 570000, "rew": 27.5395, "rew_std": 13.1816, "Agent": "A2C"}, {"env_step": 600000, "rew": 32.1327, "rew_std": 4.1747, "Agent": "A2C"}, {"env_step": 630000, "rew": 33.766, "rew_std": 3.2507, "Agent": "A2C"}, {"env_step": 660000, "rew": 33.3526, "rew_std": 2.6203, "Agent": "A2C"}, {"env_step": 690000, "rew": 34.0419, "rew_std": 3.0868, "Agent": "A2C"}, {"env_step": 720000, "rew": 32.2121, "rew_std": 4.1134, "Agent": "A2C"}, {"env_step": 750000, "rew": 34.1606, "rew_std": 3.6611, "Agent": "A2C"}, {"env_step": 780000, "rew": 34.1478, "rew_std": 4.0238, "Agent": "A2C"}, {"env_step": 810000, "rew": 35.4491, "rew_std": 2.8849, "Agent": "A2C"}, {"env_step": 840000, "rew": 34.4226, "rew_std": 3.5149, "Agent": "A2C"}, {"env_step": 870000, "rew": 35.3543, "rew_std": 4.166, "Agent": "A2C"}, {"env_step": 900000, "rew": 36.1545, "rew_std": 2.3202, "Agent": "A2C"}, {"env_step": 930000, "rew": 36.5783, "rew_std": 2.0716, "Agent": "A2C"}, {"env_step": 960000, "rew": 36.4439, "rew_std": 1.685, "Agent": "A2C"}, {"env_step": 990000, "rew": 37.6793, "rew_std": 2.2051, "Agent": "A2C"}, {"env_step": 1020000, "rew": 37.2984, "rew_std": 1.7872, "Agent": "A2C"}, {"env_step": 1050000, "rew": 38.0321, "rew_std": 1.8195, "Agent": "A2C"}, {"env_step": 1080000, "rew": 37.8466, "rew_std": 1.4684, "Agent": "A2C"}, {"env_step": 1110000, "rew": 37.8256, "rew_std": 1.1826, "Agent": "A2C"}, {"env_step": 1140000, "rew": 39.0075, "rew_std": 1.0609, "Agent": "A2C"}, {"env_step": 1170000, "rew": 39.2442, "rew_std": 1.3947, "Agent": "A2C"}, {"env_step": 1200000, "rew": 39.453, "rew_std": 2.2734, "Agent": "A2C"}, {"env_step": 1230000, "rew": 35.9649, "rew_std": 6.3833, "Agent": "A2C"}, {"env_step": 1260000, "rew": 31.5987, "rew_std": 12.7203, "Agent": "A2C"}, {"env_step": 1290000, "rew": 32.8363, "rew_std": 9.2033, "Agent": "A2C"}, {"env_step": 1320000, "rew": 37.9098, "rew_std": 1.205, "Agent": "A2C"}, {"env_step": 1350000, "rew": 38.5353, "rew_std": 2.1037, "Agent": "A2C"}, {"env_step": 1380000, "rew": 38.1807, "rew_std": 1.2005, "Agent": "A2C"}, {"env_step": 1410000, "rew": 38.4203, "rew_std": 1.5115, "Agent": "A2C"}, {"env_step": 1440000, "rew": 39.6521, "rew_std": 1.7111, "Agent": "A2C"}, {"env_step": 1470000, "rew": 40.1757, "rew_std": 1.7654, "Agent": "A2C"}, {"env_step": 1500000, "rew": 38.9241, "rew_std": 4.0098, "Agent": "A2C"}, {"env_step": 1530000, "rew": 38.8691, "rew_std": 1.0207, "Agent": "A2C"}, {"env_step": 1560000, "rew": 37.6942, "rew_std": 1.4168, "Agent": "A2C"}, {"env_step": 1590000, "rew": 37.2206, "rew_std": 2.0337, "Agent": "A2C"}, {"env_step": 1620000, "rew": 37.5777, "rew_std": 1.6754, "Agent": "A2C"}, {"env_step": 1650000, "rew": 39.0639, "rew_std": 1.3262, "Agent": "A2C"}, {"env_step": 1680000, "rew": 37.425, "rew_std": 1.987, "Agent": "A2C"}, {"env_step": 1710000, "rew": 39.4677, "rew_std": 1.3648, "Agent": "A2C"}, {"env_step": 1740000, "rew": 39.3563, "rew_std": 1.0205, "Agent": "A2C"}, {"env_step": 1770000, "rew": 38.059, "rew_std": 1.6514, "Agent": "A2C"}, {"env_step": 1800000, "rew": 38.1195, "rew_std": 1.4545, "Agent": "A2C"}, {"env_step": 1830000, "rew": 38.2468, "rew_std": 1.1224, "Agent": "A2C"}, {"env_step": 1860000, "rew": 38.7659, "rew_std": 0.9623, "Agent": "A2C"}, {"env_step": 1890000, "rew": 38.4909, "rew_std": 2.7385, "Agent": "A2C"}, {"env_step": 1920000, "rew": 39.4811, "rew_std": 1.4997, "Agent": "A2C"}, {"env_step": 1950000, "rew": 38.6882, "rew_std": 1.5165, "Agent": "A2C"}, {"env_step": 1980000, "rew": 38.6259, "rew_std": 1.3783, "Agent": "A2C"}, {"env_step": 2010000, "rew": 38.206, "rew_std": 1.0947, "Agent": "A2C"}, {"env_step": 2040000, "rew": 38.7419, "rew_std": 1.2666, "Agent": "A2C"}, {"env_step": 2070000, "rew": 38.8753, "rew_std": 1.0511, "Agent": "A2C"}, {"env_step": 2100000, "rew": 39.2905, "rew_std": 0.8688, "Agent": "A2C"}, {"env_step": 2130000, "rew": 38.5898, "rew_std": 1.1922, "Agent": "A2C"}, {"env_step": 2160000, "rew": 38.8536, "rew_std": 1.4781, "Agent": "A2C"}, {"env_step": 2190000, "rew": 38.6747, "rew_std": 1.4462, "Agent": "A2C"}, {"env_step": 2220000, "rew": 39.1438, "rew_std": 1.8135, "Agent": "A2C"}, {"env_step": 2250000, "rew": 38.1845, "rew_std": 1.669, "Agent": "A2C"}, {"env_step": 2280000, "rew": 38.0937, "rew_std": 1.3581, "Agent": "A2C"}, {"env_step": 2310000, "rew": 39.145, "rew_std": 1.1566, "Agent": "A2C"}, {"env_step": 2340000, "rew": 38.9103, "rew_std": 1.674, "Agent": "A2C"}, {"env_step": 2370000, "rew": 39.5428, "rew_std": 1.1176, "Agent": "A2C"}, {"env_step": 2400000, "rew": 39.1527, "rew_std": 1.1771, "Agent": "A2C"}, {"env_step": 2430000, "rew": 39.2441, "rew_std": 1.3234, "Agent": "A2C"}, {"env_step": 2460000, "rew": 38.2308, "rew_std": 1.3427, "Agent": "A2C"}, {"env_step": 2490000, "rew": 38.306, "rew_std": 1.3113, "Agent": "A2C"}, {"env_step": 2520000, "rew": 38.9159, "rew_std": 1.0674, "Agent": "A2C"}, {"env_step": 2550000, "rew": 38.9894, "rew_std": 1.1316, "Agent": "A2C"}, {"env_step": 2580000, "rew": 39.2214, "rew_std": 1.4829, "Agent": "A2C"}, {"env_step": 2610000, "rew": 39.6471, "rew_std": 1.1784, "Agent": "A2C"}, {"env_step": 2640000, "rew": 39.0951, "rew_std": 1.156, "Agent": "A2C"}, {"env_step": 2670000, "rew": 38.9363, "rew_std": 1.1787, "Agent": "A2C"}, {"env_step": 2700000, "rew": 38.4542, "rew_std": 1.7708, "Agent": "A2C"}, {"env_step": 2730000, "rew": 38.2105, "rew_std": 1.439, "Agent": "A2C"}, {"env_step": 2760000, "rew": 38.8404, "rew_std": 1.7091, "Agent": "A2C"}, {"env_step": 2790000, "rew": 38.7218, "rew_std": 1.588, "Agent": "A2C"}, {"env_step": 2820000, "rew": 39.4069, "rew_std": 1.5121, "Agent": "A2C"}, {"env_step": 2850000, "rew": 39.1226, "rew_std": 1.2302, "Agent": "A2C"}, {"env_step": 2880000, "rew": 38.7814, "rew_std": 1.1653, "Agent": "A2C"}, {"env_step": 2910000, "rew": 38.7925, "rew_std": 1.3412, "Agent": "A2C"}, {"env_step": 2940000, "rew": 38.8147, "rew_std": 1.3375, "Agent": "A2C"}, {"env_step": 2970000, "rew": 38.8382, "rew_std": 1.1257, "Agent": "A2C"}, {"env_step": 3000000, "rew": 38.6791, "rew_std": 1.0833, "Agent": "A2C"}] \ No newline at end of file diff --git a/examples/mujoco/benchmark/Walker2d-v3/result.json b/examples/mujoco/benchmark/Walker2d-v3/result.json new file mode 100644 index 0000000..fdc7905 --- /dev/null +++ b/examples/mujoco/benchmark/Walker2d-v3/result.json @@ -0,0 +1 @@ +[{"env_step": 0, "rew": 6.442, "rew_std": 12.7772, "Agent": "SAC"}, {"env_step": 5000, "rew": 384.2125, "rew_std": 91.1295, "Agent": "SAC"}, {"env_step": 10000, "rew": 311.4904, "rew_std": 92.4643, "Agent": "SAC"}, {"env_step": 15000, "rew": 341.5975, "rew_std": 61.666, "Agent": "SAC"}, {"env_step": 20000, "rew": 332.6649, "rew_std": 99.9772, "Agent": "SAC"}, {"env_step": 25000, "rew": 381.5184, "rew_std": 156.4263, "Agent": "SAC"}, {"env_step": 30000, "rew": 396.9952, "rew_std": 101.5131, "Agent": "SAC"}, {"env_step": 35000, "rew": 428.0762, "rew_std": 112.466, "Agent": "SAC"}, {"env_step": 40000, "rew": 538.719, "rew_std": 425.0536, "Agent": "SAC"}, {"env_step": 45000, "rew": 613.2465, "rew_std": 440.9208, "Agent": "SAC"}, {"env_step": 50000, "rew": 695.3813, "rew_std": 413.9007, "Agent": "SAC"}, {"env_step": 55000, "rew": 867.3111, "rew_std": 429.0798, "Agent": "SAC"}, {"env_step": 60000, "rew": 821.5923, "rew_std": 626.4058, "Agent": "SAC"}, {"env_step": 65000, "rew": 724.8063, "rew_std": 548.7352, "Agent": "SAC"}, {"env_step": 70000, "rew": 869.673, "rew_std": 605.068, "Agent": "SAC"}, {"env_step": 75000, "rew": 887.8157, "rew_std": 670.5667, "Agent": "SAC"}, {"env_step": 80000, "rew": 842.7084, "rew_std": 595.3462, "Agent": "SAC"}, {"env_step": 85000, "rew": 934.1402, "rew_std": 445.0447, "Agent": "SAC"}, {"env_step": 90000, "rew": 843.5223, "rew_std": 707.1264, "Agent": "SAC"}, {"env_step": 95000, "rew": 1145.929, "rew_std": 693.5426, "Agent": "SAC"}, {"env_step": 100000, "rew": 1305.246, "rew_std": 795.0855, "Agent": "SAC"}, {"env_step": 105000, "rew": 982.3865, "rew_std": 559.6456, "Agent": "SAC"}, {"env_step": 110000, "rew": 1438.316, "rew_std": 897.6387, "Agent": "SAC"}, {"env_step": 115000, "rew": 1684.2845, "rew_std": 980.9338, "Agent": "SAC"}, {"env_step": 120000, "rew": 1576.6348, "rew_std": 973.7348, "Agent": "SAC"}, {"env_step": 125000, "rew": 1635.8776, "rew_std": 1040.758, "Agent": "SAC"}, {"env_step": 130000, "rew": 1260.5658, "rew_std": 771.0769, "Agent": "SAC"}, {"env_step": 135000, "rew": 1870.8694, "rew_std": 948.5044, "Agent": "SAC"}, {"env_step": 140000, "rew": 1804.7263, "rew_std": 1012.1946, "Agent": "SAC"}, {"env_step": 145000, "rew": 1733.9893, "rew_std": 1143.5478, "Agent": "SAC"}, {"env_step": 150000, "rew": 1981.0082, "rew_std": 1216.2987, "Agent": "SAC"}, {"env_step": 155000, "rew": 2061.6254, "rew_std": 1290.993, "Agent": "SAC"}, {"env_step": 160000, "rew": 2199.765, "rew_std": 1182.4863, "Agent": "SAC"}, {"env_step": 165000, "rew": 2295.9367, "rew_std": 1338.0812, "Agent": "SAC"}, {"env_step": 170000, "rew": 2374.0511, "rew_std": 1184.7679, "Agent": "SAC"}, {"env_step": 175000, "rew": 2230.0855, "rew_std": 1255.8343, "Agent": "SAC"}, {"env_step": 180000, "rew": 2253.6271, "rew_std": 1346.9405, "Agent": "SAC"}, {"env_step": 185000, "rew": 2155.7215, "rew_std": 1412.9678, "Agent": "SAC"}, {"env_step": 190000, "rew": 2449.8378, "rew_std": 1421.6176, "Agent": "SAC"}, {"env_step": 195000, "rew": 1713.5455, "rew_std": 1319.3, "Agent": "SAC"}, {"env_step": 200000, "rew": 2530.5179, "rew_std": 1300.063, "Agent": "SAC"}, {"env_step": 205000, "rew": 2560.3269, "rew_std": 1206.713, "Agent": "SAC"}, {"env_step": 210000, "rew": 2608.8807, "rew_std": 1516.2121, "Agent": "SAC"}, {"env_step": 215000, "rew": 2940.6585, "rew_std": 1263.8721, "Agent": "SAC"}, {"env_step": 220000, "rew": 2835.1198, "rew_std": 1313.7764, "Agent": "SAC"}, {"env_step": 225000, "rew": 2795.9974, "rew_std": 1438.7164, "Agent": "SAC"}, {"env_step": 230000, "rew": 2982.4341, "rew_std": 1106.1144, "Agent": "SAC"}, {"env_step": 235000, "rew": 3005.3469, "rew_std": 1261.5498, "Agent": "SAC"}, {"env_step": 240000, "rew": 2919.6294, "rew_std": 1372.1881, "Agent": "SAC"}, {"env_step": 245000, "rew": 3230.0246, "rew_std": 964.7996, "Agent": "SAC"}, {"env_step": 250000, "rew": 3057.256, "rew_std": 1279.675, "Agent": "SAC"}, {"env_step": 255000, "rew": 2914.3693, "rew_std": 1492.3339, "Agent": "SAC"}, {"env_step": 260000, "rew": 3260.669, "rew_std": 1081.3091, "Agent": "SAC"}, {"env_step": 265000, "rew": 3395.4337, "rew_std": 1115.2453, "Agent": "SAC"}, {"env_step": 270000, "rew": 3382.5232, "rew_std": 1198.3392, "Agent": "SAC"}, {"env_step": 275000, "rew": 3343.9861, "rew_std": 1189.3488, "Agent": "SAC"}, {"env_step": 280000, "rew": 3486.7607, "rew_std": 1027.7273, "Agent": "SAC"}, {"env_step": 285000, "rew": 3407.4401, "rew_std": 1093.805, "Agent": "SAC"}, {"env_step": 290000, "rew": 3582.8111, "rew_std": 1225.0489, "Agent": "SAC"}, {"env_step": 295000, "rew": 3561.0796, "rew_std": 1143.1364, "Agent": "SAC"}, {"env_step": 300000, "rew": 3386.777, "rew_std": 1182.0262, "Agent": "SAC"}, {"env_step": 305000, "rew": 3227.8796, "rew_std": 1305.9756, "Agent": "SAC"}, {"env_step": 310000, "rew": 3755.7218, "rew_std": 908.9631, "Agent": "SAC"}, {"env_step": 315000, "rew": 3697.4154, "rew_std": 1144.1577, "Agent": "SAC"}, {"env_step": 320000, "rew": 3523.1265, "rew_std": 1185.8277, "Agent": "SAC"}, {"env_step": 325000, "rew": 3865.0648, "rew_std": 734.798, "Agent": "SAC"}, {"env_step": 330000, "rew": 3436.0228, "rew_std": 1121.3988, "Agent": "SAC"}, {"env_step": 335000, "rew": 3778.7621, "rew_std": 779.3629, "Agent": "SAC"}, {"env_step": 340000, "rew": 3853.8889, "rew_std": 784.8108, "Agent": "SAC"}, {"env_step": 345000, "rew": 3567.5159, "rew_std": 841.2435, "Agent": "SAC"}, {"env_step": 350000, "rew": 4101.7505, "rew_std": 448.9228, "Agent": "SAC"}, {"env_step": 355000, "rew": 4027.0011, "rew_std": 704.9953, "Agent": "SAC"}, {"env_step": 360000, "rew": 4103.7433, "rew_std": 525.8276, "Agent": "SAC"}, {"env_step": 365000, "rew": 4173.6105, "rew_std": 505.5541, "Agent": "SAC"}, {"env_step": 370000, "rew": 4149.5763, "rew_std": 724.6827, "Agent": "SAC"}, {"env_step": 375000, "rew": 3942.1668, "rew_std": 843.3023, "Agent": "SAC"}, {"env_step": 380000, "rew": 4152.364, "rew_std": 435.6519, "Agent": "SAC"}, {"env_step": 385000, "rew": 4189.1143, "rew_std": 475.1204, "Agent": "SAC"}, {"env_step": 390000, "rew": 4311.4203, "rew_std": 338.2765, "Agent": "SAC"}, {"env_step": 395000, "rew": 4218.0714, "rew_std": 346.1174, "Agent": "SAC"}, {"env_step": 400000, "rew": 4289.7702, "rew_std": 345.5157, "Agent": "SAC"}, {"env_step": 405000, "rew": 4251.2825, "rew_std": 417.9483, "Agent": "SAC"}, {"env_step": 410000, "rew": 4334.316, "rew_std": 352.4353, "Agent": "SAC"}, {"env_step": 415000, "rew": 4291.9864, "rew_std": 328.9922, "Agent": "SAC"}, {"env_step": 420000, "rew": 4333.559, "rew_std": 412.5937, "Agent": "SAC"}, {"env_step": 425000, "rew": 3895.0666, "rew_std": 1285.4803, "Agent": "SAC"}, {"env_step": 430000, "rew": 4089.7339, "rew_std": 976.8529, "Agent": "SAC"}, {"env_step": 435000, "rew": 4322.8754, "rew_std": 494.2115, "Agent": "SAC"}, {"env_step": 440000, "rew": 4158.0765, "rew_std": 714.8891, "Agent": "SAC"}, {"env_step": 445000, "rew": 4307.1994, "rew_std": 545.6213, "Agent": "SAC"}, {"env_step": 450000, "rew": 4471.6417, "rew_std": 460.2717, "Agent": "SAC"}, {"env_step": 455000, "rew": 4451.7838, "rew_std": 336.7615, "Agent": "SAC"}, {"env_step": 460000, "rew": 4250.6242, "rew_std": 760.2981, "Agent": "SAC"}, {"env_step": 465000, "rew": 4348.7151, "rew_std": 326.842, "Agent": "SAC"}, {"env_step": 470000, "rew": 4303.0926, "rew_std": 502.9855, "Agent": "SAC"}, {"env_step": 475000, "rew": 4474.3392, "rew_std": 296.9774, "Agent": "SAC"}, {"env_step": 480000, "rew": 4466.7216, "rew_std": 357.6031, "Agent": "SAC"}, {"env_step": 485000, "rew": 4525.9658, "rew_std": 351.0915, "Agent": "SAC"}, {"env_step": 490000, "rew": 4519.1572, "rew_std": 342.3496, "Agent": "SAC"}, {"env_step": 495000, "rew": 4336.2845, "rew_std": 617.5122, "Agent": "SAC"}, {"env_step": 500000, "rew": 4572.7087, "rew_std": 337.2333, "Agent": "SAC"}, {"env_step": 505000, "rew": 4469.0437, "rew_std": 301.8515, "Agent": "SAC"}, {"env_step": 510000, "rew": 4458.5882, "rew_std": 421.4234, "Agent": "SAC"}, {"env_step": 515000, "rew": 4375.1096, "rew_std": 746.4311, "Agent": "SAC"}, {"env_step": 520000, "rew": 3986.7109, "rew_std": 1411.5229, "Agent": "SAC"}, {"env_step": 525000, "rew": 3944.5748, "rew_std": 1475.4799, "Agent": "SAC"}, {"env_step": 530000, "rew": 4416.7452, "rew_std": 427.131, "Agent": "SAC"}, {"env_step": 535000, "rew": 4592.5295, "rew_std": 308.1874, "Agent": "SAC"}, {"env_step": 540000, "rew": 4173.1087, "rew_std": 1166.0453, "Agent": "SAC"}, {"env_step": 545000, "rew": 4636.8842, "rew_std": 286.4875, "Agent": "SAC"}, {"env_step": 550000, "rew": 4578.3448, "rew_std": 302.9395, "Agent": "SAC"}, {"env_step": 555000, "rew": 4413.4363, "rew_std": 685.7219, "Agent": "SAC"}, {"env_step": 560000, "rew": 4584.8782, "rew_std": 372.8659, "Agent": "SAC"}, {"env_step": 565000, "rew": 4636.6749, "rew_std": 284.3277, "Agent": "SAC"}, {"env_step": 570000, "rew": 4317.3934, "rew_std": 860.2242, "Agent": "SAC"}, {"env_step": 575000, "rew": 4374.3222, "rew_std": 1105.2963, "Agent": "SAC"}, {"env_step": 580000, "rew": 4571.8318, "rew_std": 298.5848, "Agent": "SAC"}, {"env_step": 585000, "rew": 4632.2289, "rew_std": 310.1531, "Agent": "SAC"}, {"env_step": 590000, "rew": 4511.29, "rew_std": 467.326, "Agent": "SAC"}, {"env_step": 595000, "rew": 4607.8163, "rew_std": 331.9071, "Agent": "SAC"}, {"env_step": 600000, "rew": 4323.4754, "rew_std": 781.4137, "Agent": "SAC"}, {"env_step": 605000, "rew": 4671.2125, "rew_std": 285.3162, "Agent": "SAC"}, {"env_step": 610000, "rew": 4643.3239, "rew_std": 344.1169, "Agent": "SAC"}, {"env_step": 615000, "rew": 4690.2549, "rew_std": 282.8784, "Agent": "SAC"}, {"env_step": 620000, "rew": 4686.0131, "rew_std": 369.3955, "Agent": "SAC"}, {"env_step": 625000, "rew": 4688.8255, "rew_std": 324.6271, "Agent": "SAC"}, {"env_step": 630000, "rew": 4506.0534, "rew_std": 567.9693, "Agent": "SAC"}, {"env_step": 635000, "rew": 4606.6157, "rew_std": 325.8416, "Agent": "SAC"}, {"env_step": 640000, "rew": 4673.6359, "rew_std": 304.2885, "Agent": "SAC"}, {"env_step": 645000, "rew": 4624.5667, "rew_std": 273.8858, "Agent": "SAC"}, {"env_step": 650000, "rew": 4723.6719, "rew_std": 298.1089, "Agent": "SAC"}, {"env_step": 655000, "rew": 4753.7342, "rew_std": 243.8481, "Agent": "SAC"}, {"env_step": 660000, "rew": 4585.5831, "rew_std": 438.7594, "Agent": "SAC"}, {"env_step": 665000, "rew": 4783.7243, "rew_std": 288.8173, "Agent": "SAC"}, {"env_step": 670000, "rew": 4760.5305, "rew_std": 216.8594, "Agent": "SAC"}, {"env_step": 675000, "rew": 4745.5199, "rew_std": 292.853, "Agent": "SAC"}, {"env_step": 680000, "rew": 4754.3869, "rew_std": 299.4585, "Agent": "SAC"}, {"env_step": 685000, "rew": 4797.6591, "rew_std": 328.1475, "Agent": "SAC"}, {"env_step": 690000, "rew": 4749.7072, "rew_std": 259.9067, "Agent": "SAC"}, {"env_step": 695000, "rew": 4800.2761, "rew_std": 266.7347, "Agent": "SAC"}, {"env_step": 700000, "rew": 4792.0478, "rew_std": 325.5644, "Agent": "SAC"}, {"env_step": 705000, "rew": 4781.8091, "rew_std": 319.7762, "Agent": "SAC"}, {"env_step": 710000, "rew": 4755.8539, "rew_std": 301.7209, "Agent": "SAC"}, {"env_step": 715000, "rew": 4801.6133, "rew_std": 313.9985, "Agent": "SAC"}, {"env_step": 720000, "rew": 4813.7196, "rew_std": 297.5344, "Agent": "SAC"}, {"env_step": 725000, "rew": 4342.3206, "rew_std": 1459.6998, "Agent": "SAC"}, {"env_step": 730000, "rew": 4871.0308, "rew_std": 294.6682, "Agent": "SAC"}, {"env_step": 735000, "rew": 4804.8159, "rew_std": 229.0327, "Agent": "SAC"}, {"env_step": 740000, "rew": 4774.688, "rew_std": 471.3831, "Agent": "SAC"}, {"env_step": 745000, "rew": 4776.7827, "rew_std": 276.7201, "Agent": "SAC"}, {"env_step": 750000, "rew": 4874.8524, "rew_std": 272.0392, "Agent": "SAC"}, {"env_step": 755000, "rew": 4828.406, "rew_std": 272.7295, "Agent": "SAC"}, {"env_step": 760000, "rew": 4839.0527, "rew_std": 321.5127, "Agent": "SAC"}, {"env_step": 765000, "rew": 4813.0792, "rew_std": 246.6873, "Agent": "SAC"}, {"env_step": 770000, "rew": 4793.6444, "rew_std": 236.0907, "Agent": "SAC"}, {"env_step": 775000, "rew": 4351.0101, "rew_std": 1468.2281, "Agent": "SAC"}, {"env_step": 780000, "rew": 4871.9559, "rew_std": 274.0046, "Agent": "SAC"}, {"env_step": 785000, "rew": 4870.5481, "rew_std": 302.3448, "Agent": "SAC"}, {"env_step": 790000, "rew": 4810.3993, "rew_std": 290.871, "Agent": "SAC"}, {"env_step": 795000, "rew": 4869.6752, "rew_std": 320.4274, "Agent": "SAC"}, {"env_step": 800000, "rew": 4929.2482, "rew_std": 281.3255, "Agent": "SAC"}, {"env_step": 805000, "rew": 4904.1074, "rew_std": 319.2952, "Agent": "SAC"}, {"env_step": 810000, "rew": 4828.2516, "rew_std": 248.9182, "Agent": "SAC"}, {"env_step": 815000, "rew": 4868.561, "rew_std": 353.0873, "Agent": "SAC"}, {"env_step": 820000, "rew": 4852.1619, "rew_std": 316.7407, "Agent": "SAC"}, {"env_step": 825000, "rew": 4888.9612, "rew_std": 304.478, "Agent": "SAC"}, {"env_step": 830000, "rew": 4879.1963, "rew_std": 323.2041, "Agent": "SAC"}, {"env_step": 835000, "rew": 4932.074, "rew_std": 269.818, "Agent": "SAC"}, {"env_step": 840000, "rew": 4865.5276, "rew_std": 266.335, "Agent": "SAC"}, {"env_step": 845000, "rew": 4919.0112, "rew_std": 288.4665, "Agent": "SAC"}, {"env_step": 850000, "rew": 4863.3539, "rew_std": 308.8071, "Agent": "SAC"}, {"env_step": 855000, "rew": 4487.3525, "rew_std": 1507.675, "Agent": "SAC"}, {"env_step": 860000, "rew": 4595.9993, "rew_std": 1208.7824, "Agent": "SAC"}, {"env_step": 865000, "rew": 4586.4829, "rew_std": 1208.126, "Agent": "SAC"}, {"env_step": 870000, "rew": 4624.6886, "rew_std": 1072.1255, "Agent": "SAC"}, {"env_step": 875000, "rew": 4927.1432, "rew_std": 333.9255, "Agent": "SAC"}, {"env_step": 880000, "rew": 4713.0344, "rew_std": 505.1897, "Agent": "SAC"}, {"env_step": 885000, "rew": 4863.7618, "rew_std": 289.8723, "Agent": "SAC"}, {"env_step": 890000, "rew": 4906.5223, "rew_std": 309.4497, "Agent": "SAC"}, {"env_step": 895000, "rew": 4903.552, "rew_std": 334.2238, "Agent": "SAC"}, {"env_step": 900000, "rew": 4975.752, "rew_std": 204.8283, "Agent": "SAC"}, {"env_step": 905000, "rew": 4939.0452, "rew_std": 294.1009, "Agent": "SAC"}, {"env_step": 910000, "rew": 4932.2537, "rew_std": 266.7105, "Agent": "SAC"}, {"env_step": 915000, "rew": 4923.4304, "rew_std": 286.5257, "Agent": "SAC"}, {"env_step": 920000, "rew": 4925.4901, "rew_std": 321.2685, "Agent": "SAC"}, {"env_step": 925000, "rew": 4930.469, "rew_std": 258.552, "Agent": "SAC"}, {"env_step": 930000, "rew": 4956.2111, "rew_std": 247.6533, "Agent": "SAC"}, {"env_step": 935000, "rew": 4976.9714, "rew_std": 262.3986, "Agent": "SAC"}, {"env_step": 940000, "rew": 4939.6174, "rew_std": 303.659, "Agent": "SAC"}, {"env_step": 945000, "rew": 4902.8756, "rew_std": 302.1218, "Agent": "SAC"}, {"env_step": 950000, "rew": 4922.0174, "rew_std": 275.016, "Agent": "SAC"}, {"env_step": 955000, "rew": 4709.7878, "rew_std": 763.3719, "Agent": "SAC"}, {"env_step": 960000, "rew": 4433.3008, "rew_std": 1511.4133, "Agent": "SAC"}, {"env_step": 965000, "rew": 4950.9803, "rew_std": 334.9916, "Agent": "SAC"}, {"env_step": 970000, "rew": 4717.4681, "rew_std": 735.1069, "Agent": "SAC"}, {"env_step": 975000, "rew": 4871.5042, "rew_std": 302.9081, "Agent": "SAC"}, {"env_step": 980000, "rew": 4969.5632, "rew_std": 283.0509, "Agent": "SAC"}, {"env_step": 985000, "rew": 4919.6581, "rew_std": 299.977, "Agent": "SAC"}, {"env_step": 990000, "rew": 4908.7273, "rew_std": 277.1986, "Agent": "SAC"}, {"env_step": 995000, "rew": 5006.9909, "rew_std": 251.5363, "Agent": "SAC"}, {"env_step": 1000000, "rew": 4960.8519, "rew_std": 326.6731, "Agent": "SAC"}, {"env_step": 1005000, "rew": 5007.6406, "rew_std": 276.5167, "Agent": "SAC"}, {"env_step": 1010000, "rew": 4975.1938, "rew_std": 251.1375, "Agent": "SAC"}, {"env_step": 1015000, "rew": 4644.0452, "rew_std": 1054.6104, "Agent": "SAC"}, {"env_step": 1020000, "rew": 5055.6218, "rew_std": 239.9198, "Agent": "SAC"}, {"env_step": 1025000, "rew": 5011.8421, "rew_std": 264.9829, "Agent": "SAC"}, {"env_step": 1030000, "rew": 4997.3616, "rew_std": 280.4018, "Agent": "SAC"}, {"env_step": 1035000, "rew": 4966.7413, "rew_std": 272.2318, "Agent": "SAC"}, {"env_step": 1040000, "rew": 4982.7577, "rew_std": 273.0634, "Agent": "SAC"}, {"env_step": 1045000, "rew": 5036.827, "rew_std": 234.8077, "Agent": "SAC"}, {"env_step": 1050000, "rew": 5064.187, "rew_std": 270.0732, "Agent": "SAC"}, {"env_step": 1055000, "rew": 5039.7243, "rew_std": 234.6565, "Agent": "SAC"}, {"env_step": 1060000, "rew": 5048.1365, "rew_std": 267.1999, "Agent": "SAC"}, {"env_step": 1065000, "rew": 5048.2367, "rew_std": 241.2245, "Agent": "SAC"}, {"env_step": 1070000, "rew": 5020.6526, "rew_std": 246.4498, "Agent": "SAC"}, {"env_step": 1075000, "rew": 5028.6578, "rew_std": 276.1192, "Agent": "SAC"}, {"env_step": 1080000, "rew": 5055.9845, "rew_std": 285.7831, "Agent": "SAC"}, {"env_step": 1085000, "rew": 5032.9481, "rew_std": 287.6907, "Agent": "SAC"}, {"env_step": 1090000, "rew": 5094.031, "rew_std": 235.2431, "Agent": "SAC"}, {"env_step": 1095000, "rew": 5034.7263, "rew_std": 287.0603, "Agent": "SAC"}, {"env_step": 1100000, "rew": 5088.3686, "rew_std": 269.512, "Agent": "SAC"}, {"env_step": 1105000, "rew": 5070.0633, "rew_std": 300.3387, "Agent": "SAC"}, {"env_step": 1110000, "rew": 5089.1961, "rew_std": 189.3168, "Agent": "SAC"}, {"env_step": 1115000, "rew": 5099.9245, "rew_std": 250.309, "Agent": "SAC"}, {"env_step": 1120000, "rew": 5089.3528, "rew_std": 220.9666, "Agent": "SAC"}, {"env_step": 1125000, "rew": 5070.1846, "rew_std": 226.8031, "Agent": "SAC"}, {"env_step": 1130000, "rew": 5107.3689, "rew_std": 237.5596, "Agent": "SAC"}, {"env_step": 1135000, "rew": 5061.5314, "rew_std": 226.2364, "Agent": "SAC"}, {"env_step": 1140000, "rew": 5079.4029, "rew_std": 264.8791, "Agent": "SAC"}, {"env_step": 1145000, "rew": 5058.1592, "rew_std": 264.707, "Agent": "SAC"}, {"env_step": 1150000, "rew": 4580.0414, "rew_std": 1525.9135, "Agent": "SAC"}, {"env_step": 1155000, "rew": 5124.2839, "rew_std": 237.1135, "Agent": "SAC"}, {"env_step": 1160000, "rew": 4789.3126, "rew_std": 964.1734, "Agent": "SAC"}, {"env_step": 1165000, "rew": 5098.6035, "rew_std": 246.2682, "Agent": "SAC"}, {"env_step": 1170000, "rew": 4985.6138, "rew_std": 263.8668, "Agent": "SAC"}, {"env_step": 1175000, "rew": 4649.6205, "rew_std": 1425.5773, "Agent": "SAC"}, {"env_step": 1180000, "rew": 4694.9734, "rew_std": 1563.7542, "Agent": "SAC"}, {"env_step": 1185000, "rew": 4744.1841, "rew_std": 1237.5092, "Agent": "SAC"}, {"env_step": 1190000, "rew": 5118.0235, "rew_std": 295.8218, "Agent": "SAC"}, {"env_step": 1195000, "rew": 5149.0656, "rew_std": 294.5632, "Agent": "SAC"}, {"env_step": 1200000, "rew": 5146.3338, "rew_std": 256.8307, "Agent": "SAC"}, {"env_step": 1205000, "rew": 5162.8329, "rew_std": 275.7237, "Agent": "SAC"}, {"env_step": 1210000, "rew": 5160.506, "rew_std": 231.5942, "Agent": "SAC"}, {"env_step": 1215000, "rew": 5146.5724, "rew_std": 247.9617, "Agent": "SAC"}, {"env_step": 1220000, "rew": 4940.8572, "rew_std": 608.9754, "Agent": "SAC"}, {"env_step": 1225000, "rew": 5140.356, "rew_std": 255.4533, "Agent": "SAC"}, {"env_step": 1230000, "rew": 5117.0055, "rew_std": 278.9903, "Agent": "SAC"}, {"env_step": 1235000, "rew": 5098.2526, "rew_std": 242.4591, "Agent": "SAC"}, {"env_step": 1240000, "rew": 5123.496, "rew_std": 267.9716, "Agent": "SAC"}, {"env_step": 1245000, "rew": 5111.109, "rew_std": 260.9796, "Agent": "SAC"}, {"env_step": 1250000, "rew": 4649.8175, "rew_std": 1331.5951, "Agent": "SAC"}, {"env_step": 0, "rew": 6.442, "rew_std": 12.7772, "Agent": "DDPG"}, {"env_step": 5000, "rew": 209.6345, "rew_std": 106.9195, "Agent": "DDPG"}, {"env_step": 10000, "rew": 92.4845, "rew_std": 86.3134, "Agent": "DDPG"}, {"env_step": 15000, "rew": 110.9192, "rew_std": 72.742, "Agent": "DDPG"}, {"env_step": 20000, "rew": 76.0794, "rew_std": 38.337, "Agent": "DDPG"}, {"env_step": 25000, "rew": 75.6129, "rew_std": 47.0657, "Agent": "DDPG"}, {"env_step": 30000, "rew": 52.394, "rew_std": 41.8502, "Agent": "DDPG"}, {"env_step": 35000, "rew": 72.7174, "rew_std": 59.2642, "Agent": "DDPG"}, {"env_step": 40000, "rew": 76.7715, "rew_std": 37.4037, "Agent": "DDPG"}, {"env_step": 45000, "rew": 86.3831, "rew_std": 52.4139, "Agent": "DDPG"}, {"env_step": 50000, "rew": 72.4621, "rew_std": 67.6108, "Agent": "DDPG"}, {"env_step": 55000, "rew": 113.8373, "rew_std": 73.6839, "Agent": "DDPG"}, {"env_step": 60000, "rew": 109.5331, "rew_std": 53.8501, "Agent": "DDPG"}, {"env_step": 65000, "rew": 112.6556, "rew_std": 107.7279, "Agent": "DDPG"}, {"env_step": 70000, "rew": 98.8362, "rew_std": 77.1782, "Agent": "DDPG"}, {"env_step": 75000, "rew": 85.5219, "rew_std": 96.1378, "Agent": "DDPG"}, {"env_step": 80000, "rew": 76.2617, "rew_std": 76.3135, "Agent": "DDPG"}, {"env_step": 85000, "rew": 80.9462, "rew_std": 66.6375, "Agent": "DDPG"}, {"env_step": 90000, "rew": 154.9533, "rew_std": 81.1592, "Agent": "DDPG"}, {"env_step": 95000, "rew": 82.6995, "rew_std": 63.5277, "Agent": "DDPG"}, {"env_step": 100000, "rew": 112.936, "rew_std": 77.8394, "Agent": "DDPG"}, {"env_step": 105000, "rew": 78.5565, "rew_std": 60.9083, "Agent": "DDPG"}, {"env_step": 110000, "rew": 117.2462, "rew_std": 132.1358, "Agent": "DDPG"}, {"env_step": 115000, "rew": 124.6421, "rew_std": 51.137, "Agent": "DDPG"}, {"env_step": 120000, "rew": 124.3216, "rew_std": 54.4929, "Agent": "DDPG"}, {"env_step": 125000, "rew": 154.0886, "rew_std": 85.1201, "Agent": "DDPG"}, {"env_step": 130000, "rew": 104.8898, "rew_std": 57.6905, "Agent": "DDPG"}, {"env_step": 135000, "rew": 137.9333, "rew_std": 102.3852, "Agent": "DDPG"}, {"env_step": 140000, "rew": 104.9934, "rew_std": 183.7926, "Agent": "DDPG"}, {"env_step": 145000, "rew": 100.7575, "rew_std": 171.5016, "Agent": "DDPG"}, {"env_step": 150000, "rew": 90.4641, "rew_std": 68.0479, "Agent": "DDPG"}, {"env_step": 155000, "rew": 84.1395, "rew_std": 77.7355, "Agent": "DDPG"}, {"env_step": 160000, "rew": 177.786, "rew_std": 172.6902, "Agent": "DDPG"}, {"env_step": 165000, "rew": 122.3961, "rew_std": 66.4944, "Agent": "DDPG"}, {"env_step": 170000, "rew": 131.1034, "rew_std": 77.2705, "Agent": "DDPG"}, {"env_step": 175000, "rew": 111.1769, "rew_std": 84.1658, "Agent": "DDPG"}, {"env_step": 180000, "rew": 128.7461, "rew_std": 73.745, "Agent": "DDPG"}, {"env_step": 185000, "rew": 139.9738, "rew_std": 91.6826, "Agent": "DDPG"}, {"env_step": 190000, "rew": 127.8014, "rew_std": 46.3038, "Agent": "DDPG"}, {"env_step": 195000, "rew": 102.6062, "rew_std": 92.6227, "Agent": "DDPG"}, {"env_step": 200000, "rew": 132.3893, "rew_std": 100.3886, "Agent": "DDPG"}, {"env_step": 205000, "rew": 120.6603, "rew_std": 123.4709, "Agent": "DDPG"}, {"env_step": 210000, "rew": 142.4264, "rew_std": 75.8023, "Agent": "DDPG"}, {"env_step": 215000, "rew": 94.8948, "rew_std": 60.3429, "Agent": "DDPG"}, {"env_step": 220000, "rew": 190.8407, "rew_std": 105.0961, "Agent": "DDPG"}, {"env_step": 225000, "rew": 130.2816, "rew_std": 94.5308, "Agent": "DDPG"}, {"env_step": 230000, "rew": 134.1195, "rew_std": 63.554, "Agent": "DDPG"}, {"env_step": 235000, "rew": 115.2859, "rew_std": 84.2026, "Agent": "DDPG"}, {"env_step": 240000, "rew": 169.5648, "rew_std": 119.7629, "Agent": "DDPG"}, {"env_step": 245000, "rew": 137.7573, "rew_std": 80.8006, "Agent": "DDPG"}, {"env_step": 250000, "rew": 136.8237, "rew_std": 88.4955, "Agent": "DDPG"}, {"env_step": 255000, "rew": 175.491, "rew_std": 95.7576, "Agent": "DDPG"}, {"env_step": 260000, "rew": 70.75, "rew_std": 67.4469, "Agent": "DDPG"}, {"env_step": 265000, "rew": 192.4634, "rew_std": 89.4985, "Agent": "DDPG"}, {"env_step": 270000, "rew": 168.0737, "rew_std": 140.0323, "Agent": "DDPG"}, {"env_step": 275000, "rew": 239.2678, "rew_std": 191.3136, "Agent": "DDPG"}, {"env_step": 280000, "rew": 276.5269, "rew_std": 197.5713, "Agent": "DDPG"}, {"env_step": 285000, "rew": 198.5067, "rew_std": 115.4132, "Agent": "DDPG"}, {"env_step": 290000, "rew": 218.5237, "rew_std": 161.1736, "Agent": "DDPG"}, {"env_step": 295000, "rew": 107.6979, "rew_std": 76.2297, "Agent": "DDPG"}, {"env_step": 300000, "rew": 175.0175, "rew_std": 117.8822, "Agent": "DDPG"}, {"env_step": 305000, "rew": 182.3931, "rew_std": 106.9644, "Agent": "DDPG"}, {"env_step": 310000, "rew": 149.532, "rew_std": 81.8634, "Agent": "DDPG"}, {"env_step": 315000, "rew": 245.8521, "rew_std": 148.3624, "Agent": "DDPG"}, {"env_step": 320000, "rew": 222.2484, "rew_std": 208.8133, "Agent": "DDPG"}, {"env_step": 325000, "rew": 272.465, "rew_std": 236.4393, "Agent": "DDPG"}, {"env_step": 330000, "rew": 244.9023, "rew_std": 197.0817, "Agent": "DDPG"}, {"env_step": 335000, "rew": 299.4603, "rew_std": 127.8442, "Agent": "DDPG"}, {"env_step": 340000, "rew": 135.6976, "rew_std": 210.0048, "Agent": "DDPG"}, {"env_step": 345000, "rew": 206.761, "rew_std": 194.4503, "Agent": "DDPG"}, {"env_step": 350000, "rew": 188.1136, "rew_std": 175.2533, "Agent": "DDPG"}, {"env_step": 355000, "rew": 167.8189, "rew_std": 146.0266, "Agent": "DDPG"}, {"env_step": 360000, "rew": 274.149, "rew_std": 251.136, "Agent": "DDPG"}, {"env_step": 365000, "rew": 197.8242, "rew_std": 124.2311, "Agent": "DDPG"}, {"env_step": 370000, "rew": 322.0313, "rew_std": 276.4998, "Agent": "DDPG"}, {"env_step": 375000, "rew": 300.0051, "rew_std": 232.919, "Agent": "DDPG"}, {"env_step": 380000, "rew": 312.4954, "rew_std": 225.9516, "Agent": "DDPG"}, {"env_step": 385000, "rew": 339.0263, "rew_std": 289.9215, "Agent": "DDPG"}, {"env_step": 390000, "rew": 271.5934, "rew_std": 331.1485, "Agent": "DDPG"}, {"env_step": 395000, "rew": 273.9743, "rew_std": 217.7879, "Agent": "DDPG"}, {"env_step": 400000, "rew": 247.9654, "rew_std": 112.5402, "Agent": "DDPG"}, {"env_step": 405000, "rew": 307.8344, "rew_std": 290.5602, "Agent": "DDPG"}, {"env_step": 410000, "rew": 250.8802, "rew_std": 198.3437, "Agent": "DDPG"}, {"env_step": 415000, "rew": 264.7911, "rew_std": 257.0488, "Agent": "DDPG"}, {"env_step": 420000, "rew": 307.2985, "rew_std": 245.4246, "Agent": "DDPG"}, {"env_step": 425000, "rew": 491.5373, "rew_std": 320.2946, "Agent": "DDPG"}, {"env_step": 430000, "rew": 364.0395, "rew_std": 219.7442, "Agent": "DDPG"}, {"env_step": 435000, "rew": 280.9915, "rew_std": 170.8729, "Agent": "DDPG"}, {"env_step": 440000, "rew": 280.4218, "rew_std": 268.7534, "Agent": "DDPG"}, {"env_step": 445000, "rew": 137.9308, "rew_std": 100.7475, "Agent": "DDPG"}, {"env_step": 450000, "rew": 278.2392, "rew_std": 215.7367, "Agent": "DDPG"}, {"env_step": 455000, "rew": 203.5839, "rew_std": 217.6392, "Agent": "DDPG"}, {"env_step": 460000, "rew": 328.9681, "rew_std": 313.5935, "Agent": "DDPG"}, {"env_step": 465000, "rew": 328.8672, "rew_std": 287.9298, "Agent": "DDPG"}, {"env_step": 470000, "rew": 251.5499, "rew_std": 167.0568, "Agent": "DDPG"}, {"env_step": 475000, "rew": 341.3302, "rew_std": 296.015, "Agent": "DDPG"}, {"env_step": 480000, "rew": 370.4565, "rew_std": 291.124, "Agent": "DDPG"}, {"env_step": 485000, "rew": 393.9402, "rew_std": 285.233, "Agent": "DDPG"}, {"env_step": 490000, "rew": 328.3354, "rew_std": 206.531, "Agent": "DDPG"}, {"env_step": 495000, "rew": 229.3258, "rew_std": 120.8655, "Agent": "DDPG"}, {"env_step": 500000, "rew": 329.8083, "rew_std": 231.147, "Agent": "DDPG"}, {"env_step": 505000, "rew": 504.2143, "rew_std": 338.5162, "Agent": "DDPG"}, {"env_step": 510000, "rew": 435.8627, "rew_std": 347.7476, "Agent": "DDPG"}, {"env_step": 515000, "rew": 357.607, "rew_std": 211.3767, "Agent": "DDPG"}, {"env_step": 520000, "rew": 352.5914, "rew_std": 221.5513, "Agent": "DDPG"}, {"env_step": 525000, "rew": 590.6616, "rew_std": 294.2043, "Agent": "DDPG"}, {"env_step": 530000, "rew": 440.5676, "rew_std": 276.736, "Agent": "DDPG"}, {"env_step": 535000, "rew": 457.0677, "rew_std": 313.7813, "Agent": "DDPG"}, {"env_step": 540000, "rew": 567.1471, "rew_std": 531.5638, "Agent": "DDPG"}, {"env_step": 545000, "rew": 519.2827, "rew_std": 597.3046, "Agent": "DDPG"}, {"env_step": 550000, "rew": 325.0792, "rew_std": 248.0299, "Agent": "DDPG"}, {"env_step": 555000, "rew": 495.9099, "rew_std": 367.6617, "Agent": "DDPG"}, {"env_step": 560000, "rew": 354.0319, "rew_std": 283.1013, "Agent": "DDPG"}, {"env_step": 565000, "rew": 451.488, "rew_std": 347.1324, "Agent": "DDPG"}, {"env_step": 570000, "rew": 422.2737, "rew_std": 275.5602, "Agent": "DDPG"}, {"env_step": 575000, "rew": 386.7718, "rew_std": 263.3219, "Agent": "DDPG"}, {"env_step": 580000, "rew": 538.5896, "rew_std": 285.8722, "Agent": "DDPG"}, {"env_step": 585000, "rew": 506.4352, "rew_std": 240.8315, "Agent": "DDPG"}, {"env_step": 590000, "rew": 456.9596, "rew_std": 304.9017, "Agent": "DDPG"}, {"env_step": 595000, "rew": 471.5949, "rew_std": 215.2671, "Agent": "DDPG"}, {"env_step": 600000, "rew": 539.6254, "rew_std": 298.8542, "Agent": "DDPG"}, {"env_step": 605000, "rew": 411.5241, "rew_std": 338.0484, "Agent": "DDPG"}, {"env_step": 610000, "rew": 447.4153, "rew_std": 227.0487, "Agent": "DDPG"}, {"env_step": 615000, "rew": 371.9614, "rew_std": 235.5178, "Agent": "DDPG"}, {"env_step": 620000, "rew": 389.4932, "rew_std": 329.9028, "Agent": "DDPG"}, {"env_step": 625000, "rew": 710.1275, "rew_std": 479.455, "Agent": "DDPG"}, {"env_step": 630000, "rew": 715.731, "rew_std": 370.1965, "Agent": "DDPG"}, {"env_step": 635000, "rew": 617.3054, "rew_std": 383.7749, "Agent": "DDPG"}, {"env_step": 640000, "rew": 612.6088, "rew_std": 464.348, "Agent": "DDPG"}, {"env_step": 645000, "rew": 552.6727, "rew_std": 398.5933, "Agent": "DDPG"}, {"env_step": 650000, "rew": 639.7049, "rew_std": 586.3823, "Agent": "DDPG"}, {"env_step": 655000, "rew": 605.128, "rew_std": 455.4957, "Agent": "DDPG"}, {"env_step": 660000, "rew": 585.7066, "rew_std": 371.6375, "Agent": "DDPG"}, {"env_step": 665000, "rew": 637.137, "rew_std": 506.6573, "Agent": "DDPG"}, {"env_step": 670000, "rew": 496.4376, "rew_std": 337.2977, "Agent": "DDPG"}, {"env_step": 675000, "rew": 750.3062, "rew_std": 679.6938, "Agent": "DDPG"}, {"env_step": 680000, "rew": 707.022, "rew_std": 408.6625, "Agent": "DDPG"}, {"env_step": 685000, "rew": 649.8468, "rew_std": 318.9338, "Agent": "DDPG"}, {"env_step": 690000, "rew": 675.1389, "rew_std": 360.0423, "Agent": "DDPG"}, {"env_step": 695000, "rew": 1035.521, "rew_std": 602.2645, "Agent": "DDPG"}, {"env_step": 700000, "rew": 621.3213, "rew_std": 397.6714, "Agent": "DDPG"}, {"env_step": 705000, "rew": 720.6641, "rew_std": 472.6936, "Agent": "DDPG"}, {"env_step": 710000, "rew": 928.2668, "rew_std": 491.9258, "Agent": "DDPG"}, {"env_step": 715000, "rew": 720.6522, "rew_std": 438.0118, "Agent": "DDPG"}, {"env_step": 720000, "rew": 1006.4925, "rew_std": 778.8225, "Agent": "DDPG"}, {"env_step": 725000, "rew": 775.1715, "rew_std": 499.7838, "Agent": "DDPG"}, {"env_step": 730000, "rew": 734.8001, "rew_std": 535.1972, "Agent": "DDPG"}, {"env_step": 735000, "rew": 755.3114, "rew_std": 749.0306, "Agent": "DDPG"}, {"env_step": 740000, "rew": 562.1704, "rew_std": 396.1291, "Agent": "DDPG"}, {"env_step": 745000, "rew": 552.7935, "rew_std": 202.8772, "Agent": "DDPG"}, {"env_step": 750000, "rew": 805.4595, "rew_std": 345.2212, "Agent": "DDPG"}, {"env_step": 755000, "rew": 695.7063, "rew_std": 417.3729, "Agent": "DDPG"}, {"env_step": 760000, "rew": 723.2767, "rew_std": 386.8996, "Agent": "DDPG"}, {"env_step": 765000, "rew": 849.0509, "rew_std": 304.5712, "Agent": "DDPG"}, {"env_step": 770000, "rew": 784.1156, "rew_std": 299.4126, "Agent": "DDPG"}, {"env_step": 775000, "rew": 921.9506, "rew_std": 463.5833, "Agent": "DDPG"}, {"env_step": 780000, "rew": 1058.706, "rew_std": 531.6714, "Agent": "DDPG"}, {"env_step": 785000, "rew": 1114.4326, "rew_std": 496.5961, "Agent": "DDPG"}, {"env_step": 790000, "rew": 1079.6687, "rew_std": 833.8941, "Agent": "DDPG"}, {"env_step": 795000, "rew": 638.847, "rew_std": 308.0488, "Agent": "DDPG"}, {"env_step": 800000, "rew": 757.8548, "rew_std": 352.2484, "Agent": "DDPG"}, {"env_step": 805000, "rew": 831.3697, "rew_std": 445.5783, "Agent": "DDPG"}, {"env_step": 810000, "rew": 858.7461, "rew_std": 319.6896, "Agent": "DDPG"}, {"env_step": 815000, "rew": 816.9652, "rew_std": 434.1336, "Agent": "DDPG"}, {"env_step": 820000, "rew": 886.6492, "rew_std": 631.2365, "Agent": "DDPG"}, {"env_step": 825000, "rew": 1049.6078, "rew_std": 652.9144, "Agent": "DDPG"}, {"env_step": 830000, "rew": 1031.1323, "rew_std": 776.5163, "Agent": "DDPG"}, {"env_step": 835000, "rew": 1095.1575, "rew_std": 512.6977, "Agent": "DDPG"}, {"env_step": 840000, "rew": 1219.7758, "rew_std": 664.5146, "Agent": "DDPG"}, {"env_step": 845000, "rew": 1011.1645, "rew_std": 535.7047, "Agent": "DDPG"}, {"env_step": 850000, "rew": 904.2512, "rew_std": 425.4184, "Agent": "DDPG"}, {"env_step": 855000, "rew": 940.3503, "rew_std": 287.5882, "Agent": "DDPG"}, {"env_step": 860000, "rew": 884.3952, "rew_std": 379.0463, "Agent": "DDPG"}, {"env_step": 865000, "rew": 817.2448, "rew_std": 341.9711, "Agent": "DDPG"}, {"env_step": 870000, "rew": 935.3364, "rew_std": 626.8482, "Agent": "DDPG"}, {"env_step": 875000, "rew": 861.8186, "rew_std": 404.6279, "Agent": "DDPG"}, {"env_step": 880000, "rew": 1105.0786, "rew_std": 457.7364, "Agent": "DDPG"}, {"env_step": 885000, "rew": 1006.9897, "rew_std": 642.6791, "Agent": "DDPG"}, {"env_step": 890000, "rew": 1145.9055, "rew_std": 788.6456, "Agent": "DDPG"}, {"env_step": 895000, "rew": 793.1493, "rew_std": 699.8714, "Agent": "DDPG"}, {"env_step": 900000, "rew": 893.9485, "rew_std": 360.6161, "Agent": "DDPG"}, {"env_step": 905000, "rew": 948.873, "rew_std": 512.409, "Agent": "DDPG"}, {"env_step": 910000, "rew": 1000.8644, "rew_std": 555.9317, "Agent": "DDPG"}, {"env_step": 915000, "rew": 1246.2144, "rew_std": 686.9246, "Agent": "DDPG"}, {"env_step": 920000, "rew": 922.9555, "rew_std": 639.8346, "Agent": "DDPG"}, {"env_step": 925000, "rew": 1034.1605, "rew_std": 590.7845, "Agent": "DDPG"}, {"env_step": 930000, "rew": 888.3659, "rew_std": 510.8527, "Agent": "DDPG"}, {"env_step": 935000, "rew": 947.3354, "rew_std": 451.528, "Agent": "DDPG"}, {"env_step": 940000, "rew": 964.2577, "rew_std": 676.8375, "Agent": "DDPG"}, {"env_step": 945000, "rew": 1074.9208, "rew_std": 786.5433, "Agent": "DDPG"}, {"env_step": 950000, "rew": 792.7072, "rew_std": 621.3961, "Agent": "DDPG"}, {"env_step": 955000, "rew": 1129.5728, "rew_std": 510.5583, "Agent": "DDPG"}, {"env_step": 960000, "rew": 1109.5555, "rew_std": 593.7077, "Agent": "DDPG"}, {"env_step": 965000, "rew": 1044.0145, "rew_std": 695.5334, "Agent": "DDPG"}, {"env_step": 970000, "rew": 1213.4942, "rew_std": 613.818, "Agent": "DDPG"}, {"env_step": 975000, "rew": 1400.6112, "rew_std": 904.9948, "Agent": "DDPG"}, {"env_step": 980000, "rew": 1307.934, "rew_std": 721.4078, "Agent": "DDPG"}, {"env_step": 985000, "rew": 1171.1626, "rew_std": 930.6772, "Agent": "DDPG"}, {"env_step": 990000, "rew": 1366.0952, "rew_std": 910.6052, "Agent": "DDPG"}, {"env_step": 995000, "rew": 1160.4298, "rew_std": 508.7525, "Agent": "DDPG"}, {"env_step": 1000000, "rew": 915.7222, "rew_std": 534.5199, "Agent": "DDPG"}, {"env_step": 1005000, "rew": 1266.544, "rew_std": 694.0691, "Agent": "DDPG"}, {"env_step": 1010000, "rew": 952.1188, "rew_std": 474.778, "Agent": "DDPG"}, {"env_step": 1015000, "rew": 1266.1654, "rew_std": 1003.0514, "Agent": "DDPG"}, {"env_step": 1020000, "rew": 1223.4316, "rew_std": 494.6038, "Agent": "DDPG"}, {"env_step": 1025000, "rew": 1151.0592, "rew_std": 740.6732, "Agent": "DDPG"}, {"env_step": 1030000, "rew": 1203.8333, "rew_std": 652.7641, "Agent": "DDPG"}, {"env_step": 1035000, "rew": 1392.2484, "rew_std": 711.8577, "Agent": "DDPG"}, {"env_step": 1040000, "rew": 1020.7799, "rew_std": 486.5689, "Agent": "DDPG"}, {"env_step": 1045000, "rew": 1367.0833, "rew_std": 705.3204, "Agent": "DDPG"}, {"env_step": 1050000, "rew": 1069.1821, "rew_std": 405.9415, "Agent": "DDPG"}, {"env_step": 1055000, "rew": 1211.1614, "rew_std": 565.3418, "Agent": "DDPG"}, {"env_step": 1060000, "rew": 1477.6099, "rew_std": 990.3152, "Agent": "DDPG"}, {"env_step": 1065000, "rew": 1213.522, "rew_std": 734.6501, "Agent": "DDPG"}, {"env_step": 1070000, "rew": 1059.1154, "rew_std": 471.9167, "Agent": "DDPG"}, {"env_step": 1075000, "rew": 1164.246, "rew_std": 547.1, "Agent": "DDPG"}, {"env_step": 1080000, "rew": 1117.6033, "rew_std": 678.8722, "Agent": "DDPG"}, {"env_step": 1085000, "rew": 967.3343, "rew_std": 451.0196, "Agent": "DDPG"}, {"env_step": 1090000, "rew": 1170.0815, "rew_std": 716.037, "Agent": "DDPG"}, {"env_step": 1095000, "rew": 1105.9619, "rew_std": 795.8141, "Agent": "DDPG"}, {"env_step": 1100000, "rew": 1355.7224, "rew_std": 659.8579, "Agent": "DDPG"}, {"env_step": 1105000, "rew": 1333.6021, "rew_std": 791.1507, "Agent": "DDPG"}, {"env_step": 1110000, "rew": 1215.6688, "rew_std": 694.2394, "Agent": "DDPG"}, {"env_step": 1115000, "rew": 1535.3212, "rew_std": 654.6559, "Agent": "DDPG"}, {"env_step": 1120000, "rew": 1266.2611, "rew_std": 524.1622, "Agent": "DDPG"}, {"env_step": 1125000, "rew": 1423.7348, "rew_std": 533.8083, "Agent": "DDPG"}, {"env_step": 1130000, "rew": 1471.318, "rew_std": 822.0757, "Agent": "DDPG"}, {"env_step": 1135000, "rew": 1524.7916, "rew_std": 446.62, "Agent": "DDPG"}, {"env_step": 1140000, "rew": 1534.0366, "rew_std": 926.9304, "Agent": "DDPG"}, {"env_step": 1145000, "rew": 1279.9772, "rew_std": 415.4775, "Agent": "DDPG"}, {"env_step": 1150000, "rew": 1126.2236, "rew_std": 614.9367, "Agent": "DDPG"}, {"env_step": 1155000, "rew": 1307.0345, "rew_std": 569.8288, "Agent": "DDPG"}, {"env_step": 1160000, "rew": 1085.1028, "rew_std": 679.3547, "Agent": "DDPG"}, {"env_step": 1165000, "rew": 1159.2904, "rew_std": 685.5038, "Agent": "DDPG"}, {"env_step": 1170000, "rew": 1253.3681, "rew_std": 514.3574, "Agent": "DDPG"}, {"env_step": 1175000, "rew": 1231.8572, "rew_std": 516.4037, "Agent": "DDPG"}, {"env_step": 1180000, "rew": 1284.062, "rew_std": 710.3769, "Agent": "DDPG"}, {"env_step": 1185000, "rew": 1684.7917, "rew_std": 781.7017, "Agent": "DDPG"}, {"env_step": 1190000, "rew": 1162.9384, "rew_std": 393.7133, "Agent": "DDPG"}, {"env_step": 1195000, "rew": 1348.3814, "rew_std": 461.711, "Agent": "DDPG"}, {"env_step": 1200000, "rew": 1296.4064, "rew_std": 646.6632, "Agent": "DDPG"}, {"env_step": 1205000, "rew": 1242.8858, "rew_std": 779.497, "Agent": "DDPG"}, {"env_step": 1210000, "rew": 1698.99, "rew_std": 558.7736, "Agent": "DDPG"}, {"env_step": 1215000, "rew": 1329.7003, "rew_std": 763.7896, "Agent": "DDPG"}, {"env_step": 1220000, "rew": 1147.4303, "rew_std": 611.0743, "Agent": "DDPG"}, {"env_step": 1225000, "rew": 1242.4779, "rew_std": 586.2073, "Agent": "DDPG"}, {"env_step": 1230000, "rew": 1454.4249, "rew_std": 531.6401, "Agent": "DDPG"}, {"env_step": 1235000, "rew": 1261.938, "rew_std": 555.3104, "Agent": "DDPG"}, {"env_step": 1240000, "rew": 1164.7458, "rew_std": 489.9345, "Agent": "DDPG"}, {"env_step": 1245000, "rew": 1441.9256, "rew_std": 566.5546, "Agent": "DDPG"}, {"env_step": 1250000, "rew": 1464.3456, "rew_std": 784.2176, "Agent": "DDPG"}, {"env_step": 0, "rew": 6.442, "rew_std": 12.7772, "Agent": "TD3"}, {"env_step": 5000, "rew": 158.7514, "rew_std": 149.0815, "Agent": "TD3"}, {"env_step": 10000, "rew": 328.0599, "rew_std": 149.4269, "Agent": "TD3"}, {"env_step": 15000, "rew": 233.9062, "rew_std": 69.258, "Agent": "TD3"}, {"env_step": 20000, "rew": 201.0709, "rew_std": 80.6006, "Agent": "TD3"}, {"env_step": 25000, "rew": 225.3551, "rew_std": 94.806, "Agent": "TD3"}, {"env_step": 30000, "rew": 239.5848, "rew_std": 96.0552, "Agent": "TD3"}, {"env_step": 35000, "rew": 269.1682, "rew_std": 32.1917, "Agent": "TD3"}, {"env_step": 40000, "rew": 320.811, "rew_std": 73.1787, "Agent": "TD3"}, {"env_step": 45000, "rew": 343.9271, "rew_std": 128.5434, "Agent": "TD3"}, {"env_step": 50000, "rew": 302.3467, "rew_std": 90.5274, "Agent": "TD3"}, {"env_step": 55000, "rew": 312.8566, "rew_std": 84.0656, "Agent": "TD3"}, {"env_step": 60000, "rew": 352.6576, "rew_std": 106.7761, "Agent": "TD3"}, {"env_step": 65000, "rew": 326.7521, "rew_std": 94.8082, "Agent": "TD3"}, {"env_step": 70000, "rew": 330.8671, "rew_std": 83.0347, "Agent": "TD3"}, {"env_step": 75000, "rew": 372.7445, "rew_std": 84.5384, "Agent": "TD3"}, {"env_step": 80000, "rew": 375.3707, "rew_std": 115.8116, "Agent": "TD3"}, {"env_step": 85000, "rew": 400.8054, "rew_std": 161.2748, "Agent": "TD3"}, {"env_step": 90000, "rew": 388.8271, "rew_std": 102.1788, "Agent": "TD3"}, {"env_step": 95000, "rew": 372.0491, "rew_std": 93.8757, "Agent": "TD3"}, {"env_step": 100000, "rew": 483.179, "rew_std": 173.8667, "Agent": "TD3"}, {"env_step": 105000, "rew": 433.9904, "rew_std": 230.4379, "Agent": "TD3"}, {"env_step": 110000, "rew": 399.904, "rew_std": 146.0952, "Agent": "TD3"}, {"env_step": 115000, "rew": 429.8596, "rew_std": 186.4218, "Agent": "TD3"}, {"env_step": 120000, "rew": 516.3706, "rew_std": 239.1313, "Agent": "TD3"}, {"env_step": 125000, "rew": 442.9991, "rew_std": 141.8523, "Agent": "TD3"}, {"env_step": 130000, "rew": 557.6177, "rew_std": 265.7382, "Agent": "TD3"}, {"env_step": 135000, "rew": 469.2168, "rew_std": 133.5503, "Agent": "TD3"}, {"env_step": 140000, "rew": 524.6685, "rew_std": 422.1725, "Agent": "TD3"}, {"env_step": 145000, "rew": 684.2732, "rew_std": 423.544, "Agent": "TD3"}, {"env_step": 150000, "rew": 734.8403, "rew_std": 482.4015, "Agent": "TD3"}, {"env_step": 155000, "rew": 683.5625, "rew_std": 305.5673, "Agent": "TD3"}, {"env_step": 160000, "rew": 611.7085, "rew_std": 144.0629, "Agent": "TD3"}, {"env_step": 165000, "rew": 706.4468, "rew_std": 359.9577, "Agent": "TD3"}, {"env_step": 170000, "rew": 601.0122, "rew_std": 169.809, "Agent": "TD3"}, {"env_step": 175000, "rew": 661.1723, "rew_std": 281.8214, "Agent": "TD3"}, {"env_step": 180000, "rew": 705.3777, "rew_std": 239.7503, "Agent": "TD3"}, {"env_step": 185000, "rew": 794.0521, "rew_std": 374.1933, "Agent": "TD3"}, {"env_step": 190000, "rew": 823.9824, "rew_std": 398.4226, "Agent": "TD3"}, {"env_step": 195000, "rew": 699.2463, "rew_std": 208.1976, "Agent": "TD3"}, {"env_step": 200000, "rew": 941.3463, "rew_std": 364.9526, "Agent": "TD3"}, {"env_step": 205000, "rew": 1112.0146, "rew_std": 558.3674, "Agent": "TD3"}, {"env_step": 210000, "rew": 841.7313, "rew_std": 344.6793, "Agent": "TD3"}, {"env_step": 215000, "rew": 855.5637, "rew_std": 342.3671, "Agent": "TD3"}, {"env_step": 220000, "rew": 1082.3426, "rew_std": 419.8525, "Agent": "TD3"}, {"env_step": 225000, "rew": 996.6779, "rew_std": 302.9571, "Agent": "TD3"}, {"env_step": 230000, "rew": 1110.0677, "rew_std": 637.767, "Agent": "TD3"}, {"env_step": 235000, "rew": 986.4518, "rew_std": 486.7762, "Agent": "TD3"}, {"env_step": 240000, "rew": 1052.8166, "rew_std": 331.3324, "Agent": "TD3"}, {"env_step": 245000, "rew": 976.0542, "rew_std": 311.9513, "Agent": "TD3"}, {"env_step": 250000, "rew": 1073.1536, "rew_std": 401.591, "Agent": "TD3"}, {"env_step": 255000, "rew": 1027.4662, "rew_std": 452.8689, "Agent": "TD3"}, {"env_step": 260000, "rew": 1255.6355, "rew_std": 366.8668, "Agent": "TD3"}, {"env_step": 265000, "rew": 1134.7905, "rew_std": 633.5713, "Agent": "TD3"}, {"env_step": 270000, "rew": 1350.4891, "rew_std": 514.0592, "Agent": "TD3"}, {"env_step": 275000, "rew": 1528.3098, "rew_std": 639.93, "Agent": "TD3"}, {"env_step": 280000, "rew": 1561.4072, "rew_std": 624.7374, "Agent": "TD3"}, {"env_step": 285000, "rew": 1394.1363, "rew_std": 755.478, "Agent": "TD3"}, {"env_step": 290000, "rew": 1487.8592, "rew_std": 619.5499, "Agent": "TD3"}, {"env_step": 295000, "rew": 1613.1424, "rew_std": 429.2336, "Agent": "TD3"}, {"env_step": 300000, "rew": 1663.4857, "rew_std": 444.9467, "Agent": "TD3"}, {"env_step": 305000, "rew": 1694.6474, "rew_std": 574.26, "Agent": "TD3"}, {"env_step": 310000, "rew": 1761.2344, "rew_std": 737.9531, "Agent": "TD3"}, {"env_step": 315000, "rew": 1737.3753, "rew_std": 690.1306, "Agent": "TD3"}, {"env_step": 320000, "rew": 1484.3802, "rew_std": 683.9933, "Agent": "TD3"}, {"env_step": 325000, "rew": 1595.0691, "rew_std": 803.0037, "Agent": "TD3"}, {"env_step": 330000, "rew": 2032.7374, "rew_std": 690.3116, "Agent": "TD3"}, {"env_step": 335000, "rew": 1934.8132, "rew_std": 666.4994, "Agent": "TD3"}, {"env_step": 340000, "rew": 1780.9275, "rew_std": 848.6981, "Agent": "TD3"}, {"env_step": 345000, "rew": 1737.634, "rew_std": 610.4062, "Agent": "TD3"}, {"env_step": 350000, "rew": 1773.278, "rew_std": 800.7065, "Agent": "TD3"}, {"env_step": 355000, "rew": 1812.198, "rew_std": 839.2482, "Agent": "TD3"}, {"env_step": 360000, "rew": 1711.6147, "rew_std": 753.6186, "Agent": "TD3"}, {"env_step": 365000, "rew": 2240.8956, "rew_std": 880.8569, "Agent": "TD3"}, {"env_step": 370000, "rew": 2087.6909, "rew_std": 902.8196, "Agent": "TD3"}, {"env_step": 375000, "rew": 2304.4965, "rew_std": 833.5611, "Agent": "TD3"}, {"env_step": 380000, "rew": 2100.6447, "rew_std": 957.1324, "Agent": "TD3"}, {"env_step": 385000, "rew": 2058.1073, "rew_std": 862.5639, "Agent": "TD3"}, {"env_step": 390000, "rew": 2363.7057, "rew_std": 771.2759, "Agent": "TD3"}, {"env_step": 395000, "rew": 2455.8049, "rew_std": 658.723, "Agent": "TD3"}, {"env_step": 400000, "rew": 2348.6439, "rew_std": 633.6437, "Agent": "TD3"}, {"env_step": 405000, "rew": 2273.554, "rew_std": 667.0168, "Agent": "TD3"}, {"env_step": 410000, "rew": 2291.7861, "rew_std": 770.9224, "Agent": "TD3"}, {"env_step": 415000, "rew": 2513.2399, "rew_std": 848.8201, "Agent": "TD3"}, {"env_step": 420000, "rew": 2208.2059, "rew_std": 696.3375, "Agent": "TD3"}, {"env_step": 425000, "rew": 2166.119, "rew_std": 715.4351, "Agent": "TD3"}, {"env_step": 430000, "rew": 2224.1711, "rew_std": 731.2473, "Agent": "TD3"}, {"env_step": 435000, "rew": 1897.0515, "rew_std": 880.6732, "Agent": "TD3"}, {"env_step": 440000, "rew": 2551.5153, "rew_std": 701.0192, "Agent": "TD3"}, {"env_step": 445000, "rew": 2334.9395, "rew_std": 703.5641, "Agent": "TD3"}, {"env_step": 450000, "rew": 2505.681, "rew_std": 686.3249, "Agent": "TD3"}, {"env_step": 455000, "rew": 2355.8402, "rew_std": 890.6842, "Agent": "TD3"}, {"env_step": 460000, "rew": 2293.5938, "rew_std": 838.344, "Agent": "TD3"}, {"env_step": 465000, "rew": 2368.7359, "rew_std": 830.1159, "Agent": "TD3"}, {"env_step": 470000, "rew": 2411.0592, "rew_std": 704.3093, "Agent": "TD3"}, {"env_step": 475000, "rew": 2751.6453, "rew_std": 705.4476, "Agent": "TD3"}, {"env_step": 480000, "rew": 2580.7633, "rew_std": 779.1095, "Agent": "TD3"}, {"env_step": 485000, "rew": 2978.5183, "rew_std": 727.8662, "Agent": "TD3"}, {"env_step": 490000, "rew": 2824.3097, "rew_std": 749.4953, "Agent": "TD3"}, {"env_step": 495000, "rew": 2485.2124, "rew_std": 769.164, "Agent": "TD3"}, {"env_step": 500000, "rew": 2854.0884, "rew_std": 661.4938, "Agent": "TD3"}, {"env_step": 505000, "rew": 2143.0347, "rew_std": 963.777, "Agent": "TD3"}, {"env_step": 510000, "rew": 3001.1841, "rew_std": 742.2609, "Agent": "TD3"}, {"env_step": 515000, "rew": 2736.8395, "rew_std": 781.8594, "Agent": "TD3"}, {"env_step": 520000, "rew": 2945.8561, "rew_std": 707.811, "Agent": "TD3"}, {"env_step": 525000, "rew": 2805.6244, "rew_std": 579.5327, "Agent": "TD3"}, {"env_step": 530000, "rew": 2796.7993, "rew_std": 681.2886, "Agent": "TD3"}, {"env_step": 535000, "rew": 2951.7061, "rew_std": 682.8467, "Agent": "TD3"}, {"env_step": 540000, "rew": 2958.4438, "rew_std": 684.3542, "Agent": "TD3"}, {"env_step": 545000, "rew": 2872.1978, "rew_std": 734.9229, "Agent": "TD3"}, {"env_step": 550000, "rew": 3019.7389, "rew_std": 756.6407, "Agent": "TD3"}, {"env_step": 555000, "rew": 3024.8354, "rew_std": 588.9253, "Agent": "TD3"}, {"env_step": 560000, "rew": 3044.8315, "rew_std": 669.2755, "Agent": "TD3"}, {"env_step": 565000, "rew": 3151.5658, "rew_std": 725.1738, "Agent": "TD3"}, {"env_step": 570000, "rew": 2815.677, "rew_std": 768.4374, "Agent": "TD3"}, {"env_step": 575000, "rew": 2930.3515, "rew_std": 882.4038, "Agent": "TD3"}, {"env_step": 580000, "rew": 3024.0144, "rew_std": 762.7719, "Agent": "TD3"}, {"env_step": 585000, "rew": 2818.3069, "rew_std": 702.1715, "Agent": "TD3"}, {"env_step": 590000, "rew": 3214.1448, "rew_std": 524.9864, "Agent": "TD3"}, {"env_step": 595000, "rew": 3194.9089, "rew_std": 602.4906, "Agent": "TD3"}, {"env_step": 600000, "rew": 3249.3552, "rew_std": 561.4048, "Agent": "TD3"}, {"env_step": 605000, "rew": 3195.2999, "rew_std": 553.2233, "Agent": "TD3"}, {"env_step": 610000, "rew": 3223.7284, "rew_std": 745.6228, "Agent": "TD3"}, {"env_step": 615000, "rew": 3494.4464, "rew_std": 460.7696, "Agent": "TD3"}, {"env_step": 620000, "rew": 3250.9882, "rew_std": 632.032, "Agent": "TD3"}, {"env_step": 625000, "rew": 3252.4065, "rew_std": 362.011, "Agent": "TD3"}, {"env_step": 630000, "rew": 3348.9026, "rew_std": 541.9568, "Agent": "TD3"}, {"env_step": 635000, "rew": 3341.5474, "rew_std": 520.4724, "Agent": "TD3"}, {"env_step": 640000, "rew": 3465.9107, "rew_std": 470.1472, "Agent": "TD3"}, {"env_step": 645000, "rew": 3447.5678, "rew_std": 339.6862, "Agent": "TD3"}, {"env_step": 650000, "rew": 3526.4283, "rew_std": 392.9445, "Agent": "TD3"}, {"env_step": 655000, "rew": 3497.1426, "rew_std": 299.4359, "Agent": "TD3"}, {"env_step": 660000, "rew": 3560.9044, "rew_std": 274.2762, "Agent": "TD3"}, {"env_step": 665000, "rew": 3481.2923, "rew_std": 508.8687, "Agent": "TD3"}, {"env_step": 670000, "rew": 3450.3515, "rew_std": 474.4117, "Agent": "TD3"}, {"env_step": 675000, "rew": 3620.2938, "rew_std": 262.9504, "Agent": "TD3"}, {"env_step": 680000, "rew": 3605.5657, "rew_std": 327.8125, "Agent": "TD3"}, {"env_step": 685000, "rew": 3693.7608, "rew_std": 285.3848, "Agent": "TD3"}, {"env_step": 690000, "rew": 3646.776, "rew_std": 251.3765, "Agent": "TD3"}, {"env_step": 695000, "rew": 3634.8418, "rew_std": 401.0298, "Agent": "TD3"}, {"env_step": 700000, "rew": 3493.2776, "rew_std": 498.0473, "Agent": "TD3"}, {"env_step": 705000, "rew": 3558.622, "rew_std": 311.7681, "Agent": "TD3"}, {"env_step": 710000, "rew": 3671.6434, "rew_std": 353.3217, "Agent": "TD3"}, {"env_step": 715000, "rew": 3455.5073, "rew_std": 537.4161, "Agent": "TD3"}, {"env_step": 720000, "rew": 3652.0814, "rew_std": 369.6568, "Agent": "TD3"}, {"env_step": 725000, "rew": 3601.9809, "rew_std": 502.9231, "Agent": "TD3"}, {"env_step": 730000, "rew": 3667.8053, "rew_std": 364.6437, "Agent": "TD3"}, {"env_step": 735000, "rew": 3734.6185, "rew_std": 379.8678, "Agent": "TD3"}, {"env_step": 740000, "rew": 3624.3996, "rew_std": 406.5599, "Agent": "TD3"}, {"env_step": 745000, "rew": 3550.2741, "rew_std": 567.7901, "Agent": "TD3"}, {"env_step": 750000, "rew": 3617.8669, "rew_std": 448.0757, "Agent": "TD3"}, {"env_step": 755000, "rew": 3741.3288, "rew_std": 326.8557, "Agent": "TD3"}, {"env_step": 760000, "rew": 3730.5887, "rew_std": 325.761, "Agent": "TD3"}, {"env_step": 765000, "rew": 3729.6041, "rew_std": 362.6786, "Agent": "TD3"}, {"env_step": 770000, "rew": 3711.055, "rew_std": 378.0649, "Agent": "TD3"}, {"env_step": 775000, "rew": 3671.1828, "rew_std": 503.9591, "Agent": "TD3"}, {"env_step": 780000, "rew": 3588.0924, "rew_std": 560.2544, "Agent": "TD3"}, {"env_step": 785000, "rew": 3698.8766, "rew_std": 416.8417, "Agent": "TD3"}, {"env_step": 790000, "rew": 3750.5026, "rew_std": 394.3008, "Agent": "TD3"}, {"env_step": 795000, "rew": 3787.7582, "rew_std": 340.0923, "Agent": "TD3"}, {"env_step": 800000, "rew": 3740.6554, "rew_std": 441.4694, "Agent": "TD3"}, {"env_step": 805000, "rew": 3659.0503, "rew_std": 441.8842, "Agent": "TD3"}, {"env_step": 810000, "rew": 3848.2124, "rew_std": 338.7611, "Agent": "TD3"}, {"env_step": 815000, "rew": 3788.2873, "rew_std": 352.9827, "Agent": "TD3"}, {"env_step": 820000, "rew": 3676.1377, "rew_std": 447.1038, "Agent": "TD3"}, {"env_step": 825000, "rew": 3655.8464, "rew_std": 500.1373, "Agent": "TD3"}, {"env_step": 830000, "rew": 3661.1425, "rew_std": 474.9795, "Agent": "TD3"}, {"env_step": 835000, "rew": 3761.5947, "rew_std": 454.3814, "Agent": "TD3"}, {"env_step": 840000, "rew": 3547.5446, "rew_std": 649.1363, "Agent": "TD3"}, {"env_step": 845000, "rew": 3782.9667, "rew_std": 393.4763, "Agent": "TD3"}, {"env_step": 850000, "rew": 3826.4287, "rew_std": 434.9649, "Agent": "TD3"}, {"env_step": 855000, "rew": 3852.4662, "rew_std": 320.955, "Agent": "TD3"}, {"env_step": 860000, "rew": 3809.3982, "rew_std": 392.2426, "Agent": "TD3"}, {"env_step": 865000, "rew": 3922.2025, "rew_std": 301.9907, "Agent": "TD3"}, {"env_step": 870000, "rew": 3897.4311, "rew_std": 291.2018, "Agent": "TD3"}, {"env_step": 875000, "rew": 3854.7866, "rew_std": 308.0508, "Agent": "TD3"}, {"env_step": 880000, "rew": 3933.806, "rew_std": 249.8089, "Agent": "TD3"}, {"env_step": 885000, "rew": 3901.5901, "rew_std": 312.7858, "Agent": "TD3"}, {"env_step": 890000, "rew": 3801.9979, "rew_std": 403.794, "Agent": "TD3"}, {"env_step": 895000, "rew": 3867.7087, "rew_std": 321.1295, "Agent": "TD3"}, {"env_step": 900000, "rew": 3822.249, "rew_std": 238.2393, "Agent": "TD3"}, {"env_step": 905000, "rew": 3964.0144, "rew_std": 256.6233, "Agent": "TD3"}, {"env_step": 910000, "rew": 3872.3271, "rew_std": 358.6703, "Agent": "TD3"}, {"env_step": 915000, "rew": 3618.269, "rew_std": 689.9244, "Agent": "TD3"}, {"env_step": 920000, "rew": 3654.5516, "rew_std": 589.5807, "Agent": "TD3"}, {"env_step": 925000, "rew": 3726.6865, "rew_std": 570.2878, "Agent": "TD3"}, {"env_step": 930000, "rew": 3660.1793, "rew_std": 822.526, "Agent": "TD3"}, {"env_step": 935000, "rew": 3924.3608, "rew_std": 309.0233, "Agent": "TD3"}, {"env_step": 940000, "rew": 3869.3083, "rew_std": 345.4325, "Agent": "TD3"}, {"env_step": 945000, "rew": 3956.3273, "rew_std": 279.5054, "Agent": "TD3"}, {"env_step": 950000, "rew": 3886.109, "rew_std": 314.0698, "Agent": "TD3"}, {"env_step": 955000, "rew": 3887.0205, "rew_std": 341.8582, "Agent": "TD3"}, {"env_step": 960000, "rew": 3709.4522, "rew_std": 516.5253, "Agent": "TD3"}, {"env_step": 965000, "rew": 3929.1202, "rew_std": 353.7442, "Agent": "TD3"}, {"env_step": 970000, "rew": 3911.8695, "rew_std": 371.1694, "Agent": "TD3"}, {"env_step": 975000, "rew": 3908.5383, "rew_std": 361.4298, "Agent": "TD3"}, {"env_step": 980000, "rew": 3859.852, "rew_std": 417.185, "Agent": "TD3"}, {"env_step": 985000, "rew": 3982.3984, "rew_std": 274.5322, "Agent": "TD3"}, {"env_step": 990000, "rew": 3958.8785, "rew_std": 290.6377, "Agent": "TD3"}, {"env_step": 995000, "rew": 3963.8609, "rew_std": 286.8225, "Agent": "TD3"}, {"env_step": 1000000, "rew": 3842.4794, "rew_std": 542.842, "Agent": "TD3"}, {"env_step": 1005000, "rew": 3974.7944, "rew_std": 309.055, "Agent": "TD3"}, {"env_step": 1010000, "rew": 4003.5151, "rew_std": 263.8881, "Agent": "TD3"}, {"env_step": 1015000, "rew": 3983.979, "rew_std": 334.1982, "Agent": "TD3"}, {"env_step": 1020000, "rew": 3935.9096, "rew_std": 310.9028, "Agent": "TD3"}, {"env_step": 1025000, "rew": 3965.1715, "rew_std": 260.2935, "Agent": "TD3"}, {"env_step": 1030000, "rew": 3957.0111, "rew_std": 289.9182, "Agent": "TD3"}, {"env_step": 1035000, "rew": 3953.037, "rew_std": 289.5479, "Agent": "TD3"}, {"env_step": 1040000, "rew": 3947.6777, "rew_std": 317.5802, "Agent": "TD3"}, {"env_step": 1045000, "rew": 3965.4811, "rew_std": 313.9896, "Agent": "TD3"}, {"env_step": 1050000, "rew": 3981.0784, "rew_std": 260.7073, "Agent": "TD3"}, {"env_step": 1055000, "rew": 3969.5636, "rew_std": 285.3478, "Agent": "TD3"}, {"env_step": 1060000, "rew": 3870.8374, "rew_std": 380.1209, "Agent": "TD3"}, {"env_step": 1065000, "rew": 3962.4485, "rew_std": 329.861, "Agent": "TD3"}, {"env_step": 1070000, "rew": 3966.8242, "rew_std": 308.1006, "Agent": "TD3"}, {"env_step": 1075000, "rew": 3760.9236, "rew_std": 612.3897, "Agent": "TD3"}, {"env_step": 1080000, "rew": 3904.4641, "rew_std": 148.3606, "Agent": "TD3"}, {"env_step": 1085000, "rew": 4023.2404, "rew_std": 308.8932, "Agent": "TD3"}, {"env_step": 1090000, "rew": 3983.7184, "rew_std": 358.6709, "Agent": "TD3"}, {"env_step": 1095000, "rew": 3818.5878, "rew_std": 671.8452, "Agent": "TD3"}, {"env_step": 1100000, "rew": 3565.3794, "rew_std": 980.5895, "Agent": "TD3"}, {"env_step": 1105000, "rew": 3992.6244, "rew_std": 344.5732, "Agent": "TD3"}, {"env_step": 1110000, "rew": 4021.0643, "rew_std": 278.6346, "Agent": "TD3"}, {"env_step": 1115000, "rew": 4055.4366, "rew_std": 245.4637, "Agent": "TD3"}, {"env_step": 1120000, "rew": 4059.6238, "rew_std": 278.8573, "Agent": "TD3"}, {"env_step": 1125000, "rew": 4077.0175, "rew_std": 286.7732, "Agent": "TD3"}, {"env_step": 1130000, "rew": 4045.8937, "rew_std": 311.3991, "Agent": "TD3"}, {"env_step": 1135000, "rew": 3912.0977, "rew_std": 395.9068, "Agent": "TD3"}, {"env_step": 1140000, "rew": 4025.0121, "rew_std": 331.3458, "Agent": "TD3"}, {"env_step": 1145000, "rew": 3989.2987, "rew_std": 464.5883, "Agent": "TD3"}, {"env_step": 1150000, "rew": 3997.7196, "rew_std": 347.4466, "Agent": "TD3"}, {"env_step": 1155000, "rew": 3773.2647, "rew_std": 761.1243, "Agent": "TD3"}, {"env_step": 1160000, "rew": 3804.4806, "rew_std": 809.5976, "Agent": "TD3"}, {"env_step": 1165000, "rew": 3905.1056, "rew_std": 407.3495, "Agent": "TD3"}, {"env_step": 1170000, "rew": 3871.2903, "rew_std": 540.1013, "Agent": "TD3"}, {"env_step": 1175000, "rew": 4088.6869, "rew_std": 261.6677, "Agent": "TD3"}, {"env_step": 1180000, "rew": 4060.7687, "rew_std": 303.4577, "Agent": "TD3"}, {"env_step": 1185000, "rew": 3922.0452, "rew_std": 467.9461, "Agent": "TD3"}, {"env_step": 1190000, "rew": 3998.1894, "rew_std": 328.7816, "Agent": "TD3"}, {"env_step": 1195000, "rew": 4083.5345, "rew_std": 278.3686, "Agent": "TD3"}, {"env_step": 1200000, "rew": 4076.6858, "rew_std": 288.7971, "Agent": "TD3"}, {"env_step": 1205000, "rew": 3645.9029, "rew_std": 1259.4616, "Agent": "TD3"}, {"env_step": 1210000, "rew": 4022.6485, "rew_std": 257.4678, "Agent": "TD3"}, {"env_step": 1215000, "rew": 4099.1548, "rew_std": 286.4948, "Agent": "TD3"}, {"env_step": 1220000, "rew": 3926.8921, "rew_std": 500.0917, "Agent": "TD3"}, {"env_step": 1225000, "rew": 4029.5039, "rew_std": 379.5839, "Agent": "TD3"}, {"env_step": 1230000, "rew": 4072.0896, "rew_std": 319.4776, "Agent": "TD3"}, {"env_step": 1235000, "rew": 4008.971, "rew_std": 424.1579, "Agent": "TD3"}, {"env_step": 1240000, "rew": 4100.0992, "rew_std": 353.1118, "Agent": "TD3"}, {"env_step": 1245000, "rew": 4101.0508, "rew_std": 312.8775, "Agent": "TD3"}, {"env_step": 1250000, "rew": 3763.9407, "rew_std": 947.9912, "Agent": "TD3"}, {"env_step": 0, "rew": -46.1947, "rew_std": 25.5804, "Agent": "TRPO"}, {"env_step": 30720, "rew": -13.9012, "rew_std": 11.913, "Agent": "TRPO"}, {"env_step": 61440, "rew": 18.0615, "rew_std": 9.0166, "Agent": "TRPO"}, {"env_step": 92160, "rew": 51.9175, "rew_std": 16.4955, "Agent": "TRPO"}, {"env_step": 122880, "rew": 108.4958, "rew_std": 33.2241, "Agent": "TRPO"}, {"env_step": 153600, "rew": 183.9944, "rew_std": 35.8262, "Agent": "TRPO"}, {"env_step": 184320, "rew": 257.4861, "rew_std": 60.051, "Agent": "TRPO"}, {"env_step": 215040, "rew": 430.4068, "rew_std": 124.2541, "Agent": "TRPO"}, {"env_step": 245760, "rew": 564.2017, "rew_std": 146.0606, "Agent": "TRPO"}, {"env_step": 276480, "rew": 682.5168, "rew_std": 122.3187, "Agent": "TRPO"}, {"env_step": 307200, "rew": 832.5033, "rew_std": 159.2968, "Agent": "TRPO"}, {"env_step": 337920, "rew": 1014.1049, "rew_std": 221.8108, "Agent": "TRPO"}, {"env_step": 368640, "rew": 1128.5618, "rew_std": 167.9213, "Agent": "TRPO"}, {"env_step": 399360, "rew": 1146.4494, "rew_std": 215.1602, "Agent": "TRPO"}, {"env_step": 430080, "rew": 1233.6374, "rew_std": 286.8604, "Agent": "TRPO"}, {"env_step": 460800, "rew": 1261.4876, "rew_std": 301.0892, "Agent": "TRPO"}, {"env_step": 491520, "rew": 1472.6865, "rew_std": 215.7148, "Agent": "TRPO"}, {"env_step": 522240, "rew": 1501.5343, "rew_std": 428.1053, "Agent": "TRPO"}, {"env_step": 552960, "rew": 1631.5161, "rew_std": 558.1692, "Agent": "TRPO"}, {"env_step": 583680, "rew": 1741.6005, "rew_std": 483.7599, "Agent": "TRPO"}, {"env_step": 614400, "rew": 1809.3667, "rew_std": 444.8329, "Agent": "TRPO"}, {"env_step": 645120, "rew": 1943.2351, "rew_std": 386.0455, "Agent": "TRPO"}, {"env_step": 675840, "rew": 2119.3593, "rew_std": 463.9853, "Agent": "TRPO"}, {"env_step": 706560, "rew": 2160.9349, "rew_std": 299.8534, "Agent": "TRPO"}, {"env_step": 737280, "rew": 2205.6584, "rew_std": 495.8226, "Agent": "TRPO"}, {"env_step": 768000, "rew": 2148.4616, "rew_std": 617.4263, "Agent": "TRPO"}, {"env_step": 798720, "rew": 2294.2347, "rew_std": 619.654, "Agent": "TRPO"}, {"env_step": 829440, "rew": 2334.3877, "rew_std": 569.8366, "Agent": "TRPO"}, {"env_step": 860160, "rew": 2373.0195, "rew_std": 635.4872, "Agent": "TRPO"}, {"env_step": 890880, "rew": 2572.1874, "rew_std": 771.2781, "Agent": "TRPO"}, {"env_step": 921600, "rew": 2568.2581, "rew_std": 539.7726, "Agent": "TRPO"}, {"env_step": 952320, "rew": 2639.2867, "rew_std": 572.4301, "Agent": "TRPO"}, {"env_step": 983040, "rew": 2866.6508, "rew_std": 707.8937, "Agent": "TRPO"}, {"env_step": 1013760, "rew": 2731.0697, "rew_std": 675.644, "Agent": "TRPO"}, {"env_step": 1044480, "rew": 2702.1965, "rew_std": 742.4807, "Agent": "TRPO"}, {"env_step": 1075200, "rew": 3001.7621, "rew_std": 688.8314, "Agent": "TRPO"}, {"env_step": 1105920, "rew": 2877.8716, "rew_std": 922.2411, "Agent": "TRPO"}, {"env_step": 1136640, "rew": 3104.9072, "rew_std": 800.8227, "Agent": "TRPO"}, {"env_step": 1167360, "rew": 2976.5528, "rew_std": 661.5153, "Agent": "TRPO"}, {"env_step": 1198080, "rew": 3196.2228, "rew_std": 707.9679, "Agent": "TRPO"}, {"env_step": 1228800, "rew": 3237.2005, "rew_std": 505.6685, "Agent": "TRPO"}, {"env_step": 1259520, "rew": 3098.3083, "rew_std": 578.4346, "Agent": "TRPO"}, {"env_step": 1290240, "rew": 3356.962, "rew_std": 786.3838, "Agent": "TRPO"}, {"env_step": 1320960, "rew": 3241.2647, "rew_std": 600.3495, "Agent": "TRPO"}, {"env_step": 1351680, "rew": 3361.1722, "rew_std": 547.5151, "Agent": "TRPO"}, {"env_step": 1382400, "rew": 3495.3879, "rew_std": 662.5387, "Agent": "TRPO"}, {"env_step": 1413120, "rew": 3388.8443, "rew_std": 537.1016, "Agent": "TRPO"}, {"env_step": 1443840, "rew": 3493.6456, "rew_std": 509.2391, "Agent": "TRPO"}, {"env_step": 1474560, "rew": 3256.3625, "rew_std": 412.1903, "Agent": "TRPO"}, {"env_step": 1505280, "rew": 3695.3862, "rew_std": 644.5284, "Agent": "TRPO"}, {"env_step": 1536000, "rew": 3685.0253, "rew_std": 537.4812, "Agent": "TRPO"}, {"env_step": 1566720, "rew": 3565.3736, "rew_std": 647.2313, "Agent": "TRPO"}, {"env_step": 1597440, "rew": 3790.5153, "rew_std": 702.1943, "Agent": "TRPO"}, {"env_step": 1628160, "rew": 3598.3267, "rew_std": 626.2541, "Agent": "TRPO"}, {"env_step": 1658880, "rew": 3970.7287, "rew_std": 647.9836, "Agent": "TRPO"}, {"env_step": 1689600, "rew": 3658.6492, "rew_std": 645.0697, "Agent": "TRPO"}, {"env_step": 1720320, "rew": 3848.0345, "rew_std": 701.1273, "Agent": "TRPO"}, {"env_step": 1751040, "rew": 4061.4513, "rew_std": 690.0289, "Agent": "TRPO"}, {"env_step": 1781760, "rew": 3807.921, "rew_std": 706.5999, "Agent": "TRPO"}, {"env_step": 1812480, "rew": 3802.0342, "rew_std": 663.7139, "Agent": "TRPO"}, {"env_step": 1843200, "rew": 3873.6662, "rew_std": 566.3833, "Agent": "TRPO"}, {"env_step": 1873920, "rew": 4038.6082, "rew_std": 518.5079, "Agent": "TRPO"}, {"env_step": 1904640, "rew": 4167.3221, "rew_std": 440.9658, "Agent": "TRPO"}, {"env_step": 1935360, "rew": 3922.2855, "rew_std": 733.8238, "Agent": "TRPO"}, {"env_step": 1966080, "rew": 4174.7307, "rew_std": 495.1849, "Agent": "TRPO"}, {"env_step": 1996800, "rew": 4155.8384, "rew_std": 477.6376, "Agent": "TRPO"}, {"env_step": 2027520, "rew": 4259.3055, "rew_std": 577.3636, "Agent": "TRPO"}, {"env_step": 2058240, "rew": 4256.1933, "rew_std": 594.2826, "Agent": "TRPO"}, {"env_step": 2088960, "rew": 4355.0669, "rew_std": 605.8978, "Agent": "TRPO"}, {"env_step": 2119680, "rew": 4297.2252, "rew_std": 474.421, "Agent": "TRPO"}, {"env_step": 2150400, "rew": 4287.4698, "rew_std": 363.7332, "Agent": "TRPO"}, {"env_step": 2181120, "rew": 4398.2137, "rew_std": 354.3603, "Agent": "TRPO"}, {"env_step": 2211840, "rew": 4413.5796, "rew_std": 460.9681, "Agent": "TRPO"}, {"env_step": 2242560, "rew": 4329.001, "rew_std": 530.2755, "Agent": "TRPO"}, {"env_step": 2273280, "rew": 4252.3464, "rew_std": 597.5882, "Agent": "TRPO"}, {"env_step": 2304000, "rew": 4462.5657, "rew_std": 495.3802, "Agent": "TRPO"}, {"env_step": 2334720, "rew": 4374.2838, "rew_std": 457.0956, "Agent": "TRPO"}, {"env_step": 2365440, "rew": 4529.4549, "rew_std": 461.1733, "Agent": "TRPO"}, {"env_step": 2396160, "rew": 4291.4468, "rew_std": 411.2142, "Agent": "TRPO"}, {"env_step": 2426880, "rew": 4430.1459, "rew_std": 601.8812, "Agent": "TRPO"}, {"env_step": 2457600, "rew": 4508.078, "rew_std": 465.0507, "Agent": "TRPO"}, {"env_step": 2488320, "rew": 4151.3796, "rew_std": 414.656, "Agent": "TRPO"}, {"env_step": 2519040, "rew": 4705.7212, "rew_std": 477.1206, "Agent": "TRPO"}, {"env_step": 2549760, "rew": 4641.3111, "rew_std": 622.6369, "Agent": "TRPO"}, {"env_step": 2580480, "rew": 4484.1452, "rew_std": 442.0811, "Agent": "TRPO"}, {"env_step": 2611200, "rew": 4658.7294, "rew_std": 432.1496, "Agent": "TRPO"}, {"env_step": 2641920, "rew": 4625.8785, "rew_std": 357.4208, "Agent": "TRPO"}, {"env_step": 2672640, "rew": 4284.9794, "rew_std": 599.7262, "Agent": "TRPO"}, {"env_step": 2703360, "rew": 4419.1972, "rew_std": 447.5325, "Agent": "TRPO"}, {"env_step": 2734080, "rew": 4746.1067, "rew_std": 580.4515, "Agent": "TRPO"}, {"env_step": 2764800, "rew": 4756.6644, "rew_std": 502.4658, "Agent": "TRPO"}, {"env_step": 2795520, "rew": 4607.7287, "rew_std": 427.1984, "Agent": "TRPO"}, {"env_step": 2826240, "rew": 4404.4363, "rew_std": 555.3297, "Agent": "TRPO"}, {"env_step": 2856960, "rew": 4816.8623, "rew_std": 528.6794, "Agent": "TRPO"}, {"env_step": 2887680, "rew": 4753.3578, "rew_std": 715.7057, "Agent": "TRPO"}, {"env_step": 2918400, "rew": 4604.763, "rew_std": 447.4458, "Agent": "TRPO"}, {"env_step": 2949120, "rew": 4800.2693, "rew_std": 602.9667, "Agent": "TRPO"}, {"env_step": 2979840, "rew": 4772.9396, "rew_std": 368.1691, "Agent": "TRPO"}, {"env_step": 3010560, "rew": 4897.0257, "rew_std": 449.1329, "Agent": "TRPO"}, {"env_step": 3041280, "rew": 4841.7814, "rew_std": 534.8951, "Agent": "TRPO"}, {"env_step": 3072000, "rew": 4604.9043, "rew_std": 580.976, "Agent": "TRPO"}, {"env_step": 0, "rew": -46.1947, "rew_std": 25.5804, "Agent": "NPG"}, {"env_step": 30720, "rew": -29.0009, "rew_std": 15.8916, "Agent": "NPG"}, {"env_step": 61440, "rew": -10.4279, "rew_std": 11.0242, "Agent": "NPG"}, {"env_step": 92160, "rew": 11.1187, "rew_std": 9.0737, "Agent": "NPG"}, {"env_step": 122880, "rew": 27.5852, "rew_std": 16.4638, "Agent": "NPG"}, {"env_step": 153600, "rew": 56.7285, "rew_std": 23.8642, "Agent": "NPG"}, {"env_step": 184320, "rew": 81.2319, "rew_std": 29.1525, "Agent": "NPG"}, {"env_step": 215040, "rew": 159.4704, "rew_std": 47.3793, "Agent": "NPG"}, {"env_step": 245760, "rew": 174.9781, "rew_std": 57.693, "Agent": "NPG"}, {"env_step": 276480, "rew": 178.6358, "rew_std": 32.807, "Agent": "NPG"}, {"env_step": 307200, "rew": 218.3212, "rew_std": 41.035, "Agent": "NPG"}, {"env_step": 337920, "rew": 368.663, "rew_std": 80.0623, "Agent": "NPG"}, {"env_step": 368640, "rew": 432.2163, "rew_std": 63.1225, "Agent": "NPG"}, {"env_step": 399360, "rew": 568.0807, "rew_std": 125.517, "Agent": "NPG"}, {"env_step": 430080, "rew": 631.1259, "rew_std": 114.6371, "Agent": "NPG"}, {"env_step": 460800, "rew": 753.5929, "rew_std": 118.5185, "Agent": "NPG"}, {"env_step": 491520, "rew": 864.7371, "rew_std": 183.4071, "Agent": "NPG"}, {"env_step": 522240, "rew": 940.2978, "rew_std": 158.8018, "Agent": "NPG"}, {"env_step": 552960, "rew": 1021.7661, "rew_std": 159.2406, "Agent": "NPG"}, {"env_step": 583680, "rew": 1131.714, "rew_std": 150.4756, "Agent": "NPG"}, {"env_step": 614400, "rew": 1201.5181, "rew_std": 164.258, "Agent": "NPG"}, {"env_step": 645120, "rew": 1343.1191, "rew_std": 184.6686, "Agent": "NPG"}, {"env_step": 675840, "rew": 1318.401, "rew_std": 229.7468, "Agent": "NPG"}, {"env_step": 706560, "rew": 1459.1159, "rew_std": 222.6298, "Agent": "NPG"}, {"env_step": 737280, "rew": 1542.0295, "rew_std": 203.9479, "Agent": "NPG"}, {"env_step": 768000, "rew": 1634.9176, "rew_std": 218.2131, "Agent": "NPG"}, {"env_step": 798720, "rew": 1692.0983, "rew_std": 343.9785, "Agent": "NPG"}, {"env_step": 829440, "rew": 1906.1507, "rew_std": 457.5102, "Agent": "NPG"}, {"env_step": 860160, "rew": 1775.1748, "rew_std": 342.0144, "Agent": "NPG"}, {"env_step": 890880, "rew": 2102.0494, "rew_std": 355.8503, "Agent": "NPG"}, {"env_step": 921600, "rew": 2046.7794, "rew_std": 340.8884, "Agent": "NPG"}, {"env_step": 952320, "rew": 2164.3512, "rew_std": 405.7482, "Agent": "NPG"}, {"env_step": 983040, "rew": 2348.312, "rew_std": 509.9401, "Agent": "NPG"}, {"env_step": 1013760, "rew": 2365.8958, "rew_std": 523.6603, "Agent": "NPG"}, {"env_step": 1044480, "rew": 2324.9164, "rew_std": 528.4526, "Agent": "NPG"}, {"env_step": 1075200, "rew": 2493.722, "rew_std": 583.8895, "Agent": "NPG"}, {"env_step": 1105920, "rew": 2638.7282, "rew_std": 650.3356, "Agent": "NPG"}, {"env_step": 1136640, "rew": 2588.3598, "rew_std": 506.4239, "Agent": "NPG"}, {"env_step": 1167360, "rew": 2657.8218, "rew_std": 542.1361, "Agent": "NPG"}, {"env_step": 1198080, "rew": 2772.0887, "rew_std": 399.8722, "Agent": "NPG"}, {"env_step": 1228800, "rew": 2983.8532, "rew_std": 405.1427, "Agent": "NPG"}, {"env_step": 1259520, "rew": 2969.912, "rew_std": 543.0224, "Agent": "NPG"}, {"env_step": 1290240, "rew": 2898.5856, "rew_std": 687.1927, "Agent": "NPG"}, {"env_step": 1320960, "rew": 2979.687, "rew_std": 685.3181, "Agent": "NPG"}, {"env_step": 1351680, "rew": 2937.7865, "rew_std": 654.2811, "Agent": "NPG"}, {"env_step": 1382400, "rew": 3180.3262, "rew_std": 502.1675, "Agent": "NPG"}, {"env_step": 1413120, "rew": 3181.4746, "rew_std": 526.9918, "Agent": "NPG"}, {"env_step": 1443840, "rew": 3186.725, "rew_std": 483.0104, "Agent": "NPG"}, {"env_step": 1474560, "rew": 3347.1624, "rew_std": 474.3212, "Agent": "NPG"}, {"env_step": 1505280, "rew": 3264.1247, "rew_std": 542.1326, "Agent": "NPG"}, {"env_step": 1536000, "rew": 3267.3737, "rew_std": 536.477, "Agent": "NPG"}, {"env_step": 1566720, "rew": 3677.709, "rew_std": 486.0857, "Agent": "NPG"}, {"env_step": 1597440, "rew": 3379.1083, "rew_std": 683.1763, "Agent": "NPG"}, {"env_step": 1628160, "rew": 3533.6653, "rew_std": 437.9623, "Agent": "NPG"}, {"env_step": 1658880, "rew": 3534.0292, "rew_std": 787.6867, "Agent": "NPG"}, {"env_step": 1689600, "rew": 3399.7589, "rew_std": 685.1419, "Agent": "NPG"}, {"env_step": 1720320, "rew": 3587.7024, "rew_std": 471.134, "Agent": "NPG"}, {"env_step": 1751040, "rew": 3560.6233, "rew_std": 616.8093, "Agent": "NPG"}, {"env_step": 1781760, "rew": 3635.7303, "rew_std": 608.4666, "Agent": "NPG"}, {"env_step": 1812480, "rew": 3713.5899, "rew_std": 802.8577, "Agent": "NPG"}, {"env_step": 1843200, "rew": 3590.908, "rew_std": 557.7779, "Agent": "NPG"}, {"env_step": 1873920, "rew": 3903.7812, "rew_std": 655.7805, "Agent": "NPG"}, {"env_step": 1904640, "rew": 3770.5075, "rew_std": 645.9229, "Agent": "NPG"}, {"env_step": 1935360, "rew": 4110.3599, "rew_std": 657.3085, "Agent": "NPG"}, {"env_step": 1966080, "rew": 3851.8914, "rew_std": 745.2079, "Agent": "NPG"}, {"env_step": 1996800, "rew": 3998.0014, "rew_std": 650.7886, "Agent": "NPG"}, {"env_step": 2027520, "rew": 3739.3657, "rew_std": 715.6727, "Agent": "NPG"}, {"env_step": 2058240, "rew": 4230.5764, "rew_std": 578.5901, "Agent": "NPG"}, {"env_step": 2088960, "rew": 4140.935, "rew_std": 624.3245, "Agent": "NPG"}, {"env_step": 2119680, "rew": 4228.9808, "rew_std": 573.9637, "Agent": "NPG"}, {"env_step": 2150400, "rew": 4142.6649, "rew_std": 419.5781, "Agent": "NPG"}, {"env_step": 2181120, "rew": 4089.8147, "rew_std": 302.9776, "Agent": "NPG"}, {"env_step": 2211840, "rew": 4276.8854, "rew_std": 674.4424, "Agent": "NPG"}, {"env_step": 2242560, "rew": 4312.1654, "rew_std": 567.5481, "Agent": "NPG"}, {"env_step": 2273280, "rew": 4224.3566, "rew_std": 445.0954, "Agent": "NPG"}, {"env_step": 2304000, "rew": 3878.7632, "rew_std": 866.3551, "Agent": "NPG"}, {"env_step": 2334720, "rew": 4001.3801, "rew_std": 478.1705, "Agent": "NPG"}, {"env_step": 2365440, "rew": 4284.5594, "rew_std": 448.8121, "Agent": "NPG"}, {"env_step": 2396160, "rew": 4406.6923, "rew_std": 516.8683, "Agent": "NPG"}, {"env_step": 2426880, "rew": 4145.9032, "rew_std": 519.4268, "Agent": "NPG"}, {"env_step": 2457600, "rew": 4241.7632, "rew_std": 685.6791, "Agent": "NPG"}, {"env_step": 2488320, "rew": 4154.1219, "rew_std": 437.1567, "Agent": "NPG"}, {"env_step": 2519040, "rew": 4495.7374, "rew_std": 449.1806, "Agent": "NPG"}, {"env_step": 2549760, "rew": 4289.7487, "rew_std": 526.7084, "Agent": "NPG"}, {"env_step": 2580480, "rew": 4246.0211, "rew_std": 530.6265, "Agent": "NPG"}, {"env_step": 2611200, "rew": 4463.8985, "rew_std": 572.8797, "Agent": "NPG"}, {"env_step": 2641920, "rew": 4298.6282, "rew_std": 665.9416, "Agent": "NPG"}, {"env_step": 2672640, "rew": 4485.5394, "rew_std": 359.9158, "Agent": "NPG"}, {"env_step": 2703360, "rew": 4820.597, "rew_std": 318.5627, "Agent": "NPG"}, {"env_step": 2734080, "rew": 4540.1727, "rew_std": 412.9545, "Agent": "NPG"}, {"env_step": 2764800, "rew": 4500.1939, "rew_std": 596.1343, "Agent": "NPG"}, {"env_step": 2795520, "rew": 4664.0692, "rew_std": 426.6286, "Agent": "NPG"}, {"env_step": 2826240, "rew": 4516.467, "rew_std": 435.656, "Agent": "NPG"}, {"env_step": 2856960, "rew": 4545.139, "rew_std": 496.411, "Agent": "NPG"}, {"env_step": 2887680, "rew": 4444.0518, "rew_std": 596.6213, "Agent": "NPG"}, {"env_step": 2918400, "rew": 4658.8307, "rew_std": 652.0561, "Agent": "NPG"}, {"env_step": 2949120, "rew": 4509.4439, "rew_std": 466.3483, "Agent": "NPG"}, {"env_step": 2979840, "rew": 4589.8375, "rew_std": 414.0363, "Agent": "NPG"}, {"env_step": 3010560, "rew": 4693.0902, "rew_std": 490.4674, "Agent": "NPG"}, {"env_step": 3041280, "rew": 4678.8152, "rew_std": 313.0054, "Agent": "NPG"}, {"env_step": 3072000, "rew": 4736.6273, "rew_std": 376.7205, "Agent": "NPG"}, {"env_step": 0, "rew": 2.854, "rew_std": 2.439, "Agent": "PPO"}, {"env_step": 30720, "rew": 261.6371, "rew_std": 31.9544, "Agent": "PPO"}, {"env_step": 61440, "rew": 337.5775, "rew_std": 33.1557, "Agent": "PPO"}, {"env_step": 92160, "rew": 362.9134, "rew_std": 58.8816, "Agent": "PPO"}, {"env_step": 122880, "rew": 425.8273, "rew_std": 60.5094, "Agent": "PPO"}, {"env_step": 153600, "rew": 553.2555, "rew_std": 116.4205, "Agent": "PPO"}, {"env_step": 184320, "rew": 715.2234, "rew_std": 271.9853, "Agent": "PPO"}, {"env_step": 215040, "rew": 905.6901, "rew_std": 427.5947, "Agent": "PPO"}, {"env_step": 245760, "rew": 1126.158, "rew_std": 465.6568, "Agent": "PPO"}, {"env_step": 276480, "rew": 1337.0027, "rew_std": 444.4031, "Agent": "PPO"}, {"env_step": 307200, "rew": 1985.784, "rew_std": 756.4732, "Agent": "PPO"}, {"env_step": 337920, "rew": 1939.7872, "rew_std": 639.0395, "Agent": "PPO"}, {"env_step": 368640, "rew": 2077.5675, "rew_std": 774.0649, "Agent": "PPO"}, {"env_step": 399360, "rew": 2344.1341, "rew_std": 944.0267, "Agent": "PPO"}, {"env_step": 430080, "rew": 2608.1656, "rew_std": 686.5428, "Agent": "PPO"}, {"env_step": 460800, "rew": 2552.6508, "rew_std": 774.8729, "Agent": "PPO"}, {"env_step": 491520, "rew": 2759.0287, "rew_std": 613.4486, "Agent": "PPO"}, {"env_step": 522240, "rew": 2954.2882, "rew_std": 641.8847, "Agent": "PPO"}, {"env_step": 552960, "rew": 2740.7866, "rew_std": 597.9693, "Agent": "PPO"}, {"env_step": 583680, "rew": 2758.2169, "rew_std": 493.9706, "Agent": "PPO"}, {"env_step": 614400, "rew": 3133.3863, "rew_std": 838.3549, "Agent": "PPO"}, {"env_step": 645120, "rew": 3025.815, "rew_std": 441.8207, "Agent": "PPO"}, {"env_step": 675840, "rew": 3230.7174, "rew_std": 628.2298, "Agent": "PPO"}, {"env_step": 706560, "rew": 3501.1757, "rew_std": 661.1069, "Agent": "PPO"}, {"env_step": 737280, "rew": 3306.1529, "rew_std": 632.7268, "Agent": "PPO"}, {"env_step": 768000, "rew": 3202.6631, "rew_std": 575.353, "Agent": "PPO"}, {"env_step": 798720, "rew": 3125.1025, "rew_std": 641.088, "Agent": "PPO"}, {"env_step": 829440, "rew": 2937.7286, "rew_std": 972.9868, "Agent": "PPO"}, {"env_step": 860160, "rew": 3507.4568, "rew_std": 1043.6589, "Agent": "PPO"}, {"env_step": 890880, "rew": 3588.5181, "rew_std": 756.6306, "Agent": "PPO"}, {"env_step": 921600, "rew": 3286.1651, "rew_std": 1041.8169, "Agent": "PPO"}, {"env_step": 952320, "rew": 3515.5619, "rew_std": 1043.1778, "Agent": "PPO"}, {"env_step": 983040, "rew": 3493.7872, "rew_std": 1024.8728, "Agent": "PPO"}, {"env_step": 1013760, "rew": 3410.2609, "rew_std": 685.6092, "Agent": "PPO"}, {"env_step": 1044480, "rew": 3390.317, "rew_std": 795.0583, "Agent": "PPO"}, {"env_step": 1075200, "rew": 3297.4848, "rew_std": 847.0978, "Agent": "PPO"}, {"env_step": 1105920, "rew": 3789.9667, "rew_std": 896.1263, "Agent": "PPO"}, {"env_step": 1136640, "rew": 3485.3889, "rew_std": 775.1673, "Agent": "PPO"}, {"env_step": 1167360, "rew": 3766.0153, "rew_std": 762.1712, "Agent": "PPO"}, {"env_step": 1198080, "rew": 3916.4919, "rew_std": 637.0181, "Agent": "PPO"}, {"env_step": 1228800, "rew": 3641.0305, "rew_std": 734.5641, "Agent": "PPO"}, {"env_step": 1259520, "rew": 3838.3772, "rew_std": 871.0735, "Agent": "PPO"}, {"env_step": 1290240, "rew": 4149.6348, "rew_std": 822.515, "Agent": "PPO"}, {"env_step": 1320960, "rew": 3820.7451, "rew_std": 824.6649, "Agent": "PPO"}, {"env_step": 1351680, "rew": 3523.6159, "rew_std": 802.2996, "Agent": "PPO"}, {"env_step": 1382400, "rew": 3594.5379, "rew_std": 649.5449, "Agent": "PPO"}, {"env_step": 1413120, "rew": 4009.9213, "rew_std": 861.1393, "Agent": "PPO"}, {"env_step": 1443840, "rew": 3700.4344, "rew_std": 932.635, "Agent": "PPO"}, {"env_step": 1474560, "rew": 3810.8253, "rew_std": 948.657, "Agent": "PPO"}, {"env_step": 1505280, "rew": 3932.9272, "rew_std": 767.4963, "Agent": "PPO"}, {"env_step": 1536000, "rew": 3571.2177, "rew_std": 882.525, "Agent": "PPO"}, {"env_step": 1566720, "rew": 3637.1949, "rew_std": 878.2077, "Agent": "PPO"}, {"env_step": 1597440, "rew": 4002.2786, "rew_std": 669.5074, "Agent": "PPO"}, {"env_step": 1628160, "rew": 3986.5518, "rew_std": 651.4945, "Agent": "PPO"}, {"env_step": 1658880, "rew": 4073.0553, "rew_std": 530.0421, "Agent": "PPO"}, {"env_step": 1689600, "rew": 4123.7576, "rew_std": 836.6608, "Agent": "PPO"}, {"env_step": 1720320, "rew": 4093.6101, "rew_std": 889.4021, "Agent": "PPO"}, {"env_step": 1751040, "rew": 4012.082, "rew_std": 834.2345, "Agent": "PPO"}, {"env_step": 1781760, "rew": 4008.7549, "rew_std": 1018.6348, "Agent": "PPO"}, {"env_step": 1812480, "rew": 4002.3804, "rew_std": 935.0496, "Agent": "PPO"}, {"env_step": 1843200, "rew": 4109.2768, "rew_std": 884.4799, "Agent": "PPO"}, {"env_step": 1873920, "rew": 3893.1839, "rew_std": 1441.2806, "Agent": "PPO"}, {"env_step": 1904640, "rew": 3784.3364, "rew_std": 949.5072, "Agent": "PPO"}, {"env_step": 1935360, "rew": 4191.4698, "rew_std": 1208.749, "Agent": "PPO"}, {"env_step": 1966080, "rew": 4018.6931, "rew_std": 904.0015, "Agent": "PPO"}, {"env_step": 1996800, "rew": 4018.0772, "rew_std": 1023.582, "Agent": "PPO"}, {"env_step": 2027520, "rew": 3936.778, "rew_std": 706.0211, "Agent": "PPO"}, {"env_step": 2058240, "rew": 4521.6965, "rew_std": 844.448, "Agent": "PPO"}, {"env_step": 2088960, "rew": 4279.1014, "rew_std": 763.1104, "Agent": "PPO"}, {"env_step": 2119680, "rew": 3928.6194, "rew_std": 801.9427, "Agent": "PPO"}, {"env_step": 2150400, "rew": 4478.9202, "rew_std": 910.9486, "Agent": "PPO"}, {"env_step": 2181120, "rew": 4536.146, "rew_std": 754.0238, "Agent": "PPO"}, {"env_step": 2211840, "rew": 3723.36, "rew_std": 1065.5419, "Agent": "PPO"}, {"env_step": 2242560, "rew": 4600.0676, "rew_std": 843.5307, "Agent": "PPO"}, {"env_step": 2273280, "rew": 4646.8615, "rew_std": 693.3327, "Agent": "PPO"}, {"env_step": 2304000, "rew": 4895.5572, "rew_std": 704.2969, "Agent": "PPO"}, {"env_step": 2334720, "rew": 4682.4576, "rew_std": 771.115, "Agent": "PPO"}, {"env_step": 2365440, "rew": 3961.3622, "rew_std": 757.2372, "Agent": "PPO"}, {"env_step": 2396160, "rew": 4342.5382, "rew_std": 756.5942, "Agent": "PPO"}, {"env_step": 2426880, "rew": 4406.6723, "rew_std": 752.6507, "Agent": "PPO"}, {"env_step": 2457600, "rew": 4461.0387, "rew_std": 660.9203, "Agent": "PPO"}, {"env_step": 2488320, "rew": 4450.4049, "rew_std": 1070.5688, "Agent": "PPO"}, {"env_step": 2519040, "rew": 4628.4549, "rew_std": 883.0679, "Agent": "PPO"}, {"env_step": 2549760, "rew": 4353.2668, "rew_std": 1022.8285, "Agent": "PPO"}, {"env_step": 2580480, "rew": 4431.4896, "rew_std": 1042.5742, "Agent": "PPO"}, {"env_step": 2611200, "rew": 4444.7746, "rew_std": 978.6624, "Agent": "PPO"}, {"env_step": 2641920, "rew": 4753.0368, "rew_std": 904.3219, "Agent": "PPO"}, {"env_step": 2672640, "rew": 4475.2381, "rew_std": 716.1777, "Agent": "PPO"}, {"env_step": 2703360, "rew": 4213.8239, "rew_std": 919.9318, "Agent": "PPO"}, {"env_step": 2734080, "rew": 4766.5872, "rew_std": 942.6024, "Agent": "PPO"}, {"env_step": 2764800, "rew": 4714.3712, "rew_std": 988.3013, "Agent": "PPO"}, {"env_step": 2795520, "rew": 4654.9762, "rew_std": 985.8355, "Agent": "PPO"}, {"env_step": 2826240, "rew": 4544.4566, "rew_std": 994.8492, "Agent": "PPO"}, {"env_step": 2856960, "rew": 4454.7327, "rew_std": 971.1529, "Agent": "PPO"}, {"env_step": 2887680, "rew": 4622.6955, "rew_std": 1030.6566, "Agent": "PPO"}, {"env_step": 2918400, "rew": 4118.5372, "rew_std": 1002.9298, "Agent": "PPO"}, {"env_step": 2949120, "rew": 4404.5734, "rew_std": 797.9952, "Agent": "PPO"}, {"env_step": 2979840, "rew": 4564.1186, "rew_std": 614.9968, "Agent": "PPO"}, {"env_step": 3010560, "rew": 4674.4358, "rew_std": 688.045, "Agent": "PPO"}, {"env_step": 3041280, "rew": 4624.0476, "rew_std": 689.6915, "Agent": "PPO"}, {"env_step": 3072000, "rew": 4369.7249, "rew_std": 694.8409, "Agent": "PPO"}, {"env_step": 0, "rew": 4.5535, "rew_std": 2.0756, "Agent": "REINFORCE"}, {"env_step": 30720, "rew": 13.6527, "rew_std": 5.6565, "Agent": "REINFORCE"}, {"env_step": 61440, "rew": 110.1682, "rew_std": 59.6794, "Agent": "REINFORCE"}, {"env_step": 92160, "rew": 223.5316, "rew_std": 29.8609, "Agent": "REINFORCE"}, {"env_step": 122880, "rew": 200.5065, "rew_std": 39.9608, "Agent": "REINFORCE"}, {"env_step": 153600, "rew": 187.5621, "rew_std": 34.827, "Agent": "REINFORCE"}, {"env_step": 184320, "rew": 235.2624, "rew_std": 32.9012, "Agent": "REINFORCE"}, {"env_step": 215040, "rew": 239.778, "rew_std": 33.4927, "Agent": "REINFORCE"}, {"env_step": 245760, "rew": 247.0896, "rew_std": 29.8488, "Agent": "REINFORCE"}, {"env_step": 276480, "rew": 258.3275, "rew_std": 29.6617, "Agent": "REINFORCE"}, {"env_step": 307200, "rew": 266.5922, "rew_std": 30.4123, "Agent": "REINFORCE"}, {"env_step": 337920, "rew": 295.2816, "rew_std": 50.3467, "Agent": "REINFORCE"}, {"env_step": 368640, "rew": 292.9666, "rew_std": 42.0559, "Agent": "REINFORCE"}, {"env_step": 399360, "rew": 299.5063, "rew_std": 38.3164, "Agent": "REINFORCE"}, {"env_step": 430080, "rew": 271.3942, "rew_std": 46.1242, "Agent": "REINFORCE"}, {"env_step": 460800, "rew": 296.9937, "rew_std": 54.9201, "Agent": "REINFORCE"}, {"env_step": 491520, "rew": 284.61, "rew_std": 59.2053, "Agent": "REINFORCE"}, {"env_step": 522240, "rew": 323.922, "rew_std": 54.241, "Agent": "REINFORCE"}, {"env_step": 552960, "rew": 325.3197, "rew_std": 57.678, "Agent": "REINFORCE"}, {"env_step": 583680, "rew": 309.106, "rew_std": 53.6707, "Agent": "REINFORCE"}, {"env_step": 614400, "rew": 340.3094, "rew_std": 54.4306, "Agent": "REINFORCE"}, {"env_step": 645120, "rew": 322.4308, "rew_std": 66.5441, "Agent": "REINFORCE"}, {"env_step": 675840, "rew": 359.9286, "rew_std": 46.8059, "Agent": "REINFORCE"}, {"env_step": 706560, "rew": 355.6872, "rew_std": 60.0439, "Agent": "REINFORCE"}, {"env_step": 737280, "rew": 354.6147, "rew_std": 39.4527, "Agent": "REINFORCE"}, {"env_step": 768000, "rew": 342.5682, "rew_std": 35.7644, "Agent": "REINFORCE"}, {"env_step": 798720, "rew": 385.1629, "rew_std": 48.8126, "Agent": "REINFORCE"}, {"env_step": 829440, "rew": 391.1747, "rew_std": 44.5626, "Agent": "REINFORCE"}, {"env_step": 860160, "rew": 346.6364, "rew_std": 65.094, "Agent": "REINFORCE"}, {"env_step": 890880, "rew": 353.0954, "rew_std": 69.2847, "Agent": "REINFORCE"}, {"env_step": 921600, "rew": 346.909, "rew_std": 62.2388, "Agent": "REINFORCE"}, {"env_step": 952320, "rew": 367.5316, "rew_std": 60.5551, "Agent": "REINFORCE"}, {"env_step": 983040, "rew": 354.1843, "rew_std": 63.154, "Agent": "REINFORCE"}, {"env_step": 1013760, "rew": 343.4204, "rew_std": 55.6947, "Agent": "REINFORCE"}, {"env_step": 1044480, "rew": 354.1547, "rew_std": 97.6597, "Agent": "REINFORCE"}, {"env_step": 1075200, "rew": 403.1631, "rew_std": 76.9285, "Agent": "REINFORCE"}, {"env_step": 1105920, "rew": 381.1449, "rew_std": 80.7015, "Agent": "REINFORCE"}, {"env_step": 1136640, "rew": 372.1611, "rew_std": 84.8949, "Agent": "REINFORCE"}, {"env_step": 1167360, "rew": 339.0372, "rew_std": 72.4904, "Agent": "REINFORCE"}, {"env_step": 1198080, "rew": 357.2748, "rew_std": 57.2974, "Agent": "REINFORCE"}, {"env_step": 1228800, "rew": 355.9174, "rew_std": 80.0236, "Agent": "REINFORCE"}, {"env_step": 1259520, "rew": 353.4319, "rew_std": 64.1911, "Agent": "REINFORCE"}, {"env_step": 1290240, "rew": 373.0621, "rew_std": 92.4906, "Agent": "REINFORCE"}, {"env_step": 1320960, "rew": 341.2523, "rew_std": 77.9283, "Agent": "REINFORCE"}, {"env_step": 1351680, "rew": 348.859, "rew_std": 60.7955, "Agent": "REINFORCE"}, {"env_step": 1382400, "rew": 355.2645, "rew_std": 77.5044, "Agent": "REINFORCE"}, {"env_step": 1413120, "rew": 356.7775, "rew_std": 88.4894, "Agent": "REINFORCE"}, {"env_step": 1443840, "rew": 329.7672, "rew_std": 67.3729, "Agent": "REINFORCE"}, {"env_step": 1474560, "rew": 335.196, "rew_std": 94.2459, "Agent": "REINFORCE"}, {"env_step": 1505280, "rew": 356.1628, "rew_std": 75.7139, "Agent": "REINFORCE"}, {"env_step": 1536000, "rew": 352.1588, "rew_std": 71.9885, "Agent": "REINFORCE"}, {"env_step": 1566720, "rew": 361.1192, "rew_std": 62.3489, "Agent": "REINFORCE"}, {"env_step": 1597440, "rew": 352.0869, "rew_std": 69.1096, "Agent": "REINFORCE"}, {"env_step": 1628160, "rew": 344.9398, "rew_std": 61.5555, "Agent": "REINFORCE"}, {"env_step": 1658880, "rew": 371.1304, "rew_std": 78.2817, "Agent": "REINFORCE"}, {"env_step": 1689600, "rew": 356.5322, "rew_std": 118.4185, "Agent": "REINFORCE"}, {"env_step": 1720320, "rew": 370.873, "rew_std": 86.1696, "Agent": "REINFORCE"}, {"env_step": 1751040, "rew": 361.6152, "rew_std": 85.3641, "Agent": "REINFORCE"}, {"env_step": 1781760, "rew": 366.3599, "rew_std": 82.2322, "Agent": "REINFORCE"}, {"env_step": 1812480, "rew": 367.3329, "rew_std": 80.3721, "Agent": "REINFORCE"}, {"env_step": 1843200, "rew": 362.9224, "rew_std": 86.4875, "Agent": "REINFORCE"}, {"env_step": 1873920, "rew": 388.4115, "rew_std": 80.8852, "Agent": "REINFORCE"}, {"env_step": 1904640, "rew": 352.6121, "rew_std": 89.2546, "Agent": "REINFORCE"}, {"env_step": 1935360, "rew": 354.0444, "rew_std": 105.3851, "Agent": "REINFORCE"}, {"env_step": 1966080, "rew": 371.3243, "rew_std": 87.666, "Agent": "REINFORCE"}, {"env_step": 1996800, "rew": 365.3113, "rew_std": 103.0233, "Agent": "REINFORCE"}, {"env_step": 2027520, "rew": 355.7672, "rew_std": 90.9894, "Agent": "REINFORCE"}, {"env_step": 2058240, "rew": 338.4174, "rew_std": 87.8862, "Agent": "REINFORCE"}, {"env_step": 2088960, "rew": 362.839, "rew_std": 92.0007, "Agent": "REINFORCE"}, {"env_step": 2119680, "rew": 368.269, "rew_std": 102.4378, "Agent": "REINFORCE"}, {"env_step": 2150400, "rew": 373.8035, "rew_std": 87.1727, "Agent": "REINFORCE"}, {"env_step": 2181120, "rew": 361.1823, "rew_std": 78.0693, "Agent": "REINFORCE"}, {"env_step": 2211840, "rew": 367.1892, "rew_std": 102.8876, "Agent": "REINFORCE"}, {"env_step": 2242560, "rew": 365.9658, "rew_std": 81.4782, "Agent": "REINFORCE"}, {"env_step": 2273280, "rew": 360.6872, "rew_std": 114.0468, "Agent": "REINFORCE"}, {"env_step": 2304000, "rew": 365.9783, "rew_std": 122.2242, "Agent": "REINFORCE"}, {"env_step": 2334720, "rew": 379.9675, "rew_std": 81.0374, "Agent": "REINFORCE"}, {"env_step": 2365440, "rew": 345.6967, "rew_std": 87.0767, "Agent": "REINFORCE"}, {"env_step": 2396160, "rew": 366.1253, "rew_std": 105.243, "Agent": "REINFORCE"}, {"env_step": 2426880, "rew": 352.4197, "rew_std": 74.3958, "Agent": "REINFORCE"}, {"env_step": 2457600, "rew": 380.5138, "rew_std": 77.7426, "Agent": "REINFORCE"}, {"env_step": 2488320, "rew": 367.3545, "rew_std": 109.7126, "Agent": "REINFORCE"}, {"env_step": 2519040, "rew": 369.4047, "rew_std": 88.9617, "Agent": "REINFORCE"}, {"env_step": 2549760, "rew": 355.9494, "rew_std": 84.9406, "Agent": "REINFORCE"}, {"env_step": 2580480, "rew": 361.6388, "rew_std": 77.2025, "Agent": "REINFORCE"}, {"env_step": 2611200, "rew": 385.6433, "rew_std": 56.6485, "Agent": "REINFORCE"}, {"env_step": 2641920, "rew": 383.2675, "rew_std": 59.8544, "Agent": "REINFORCE"}, {"env_step": 2672640, "rew": 383.3281, "rew_std": 53.5628, "Agent": "REINFORCE"}, {"env_step": 2703360, "rew": 380.186, "rew_std": 51.4936, "Agent": "REINFORCE"}, {"env_step": 2734080, "rew": 357.4304, "rew_std": 92.4873, "Agent": "REINFORCE"}, {"env_step": 2764800, "rew": 372.8371, "rew_std": 62.6374, "Agent": "REINFORCE"}, {"env_step": 2795520, "rew": 369.476, "rew_std": 93.6221, "Agent": "REINFORCE"}, {"env_step": 2826240, "rew": 377.2572, "rew_std": 53.0474, "Agent": "REINFORCE"}, {"env_step": 2856960, "rew": 387.0054, "rew_std": 83.6139, "Agent": "REINFORCE"}, {"env_step": 2887680, "rew": 411.9642, "rew_std": 52.4265, "Agent": "REINFORCE"}, {"env_step": 2918400, "rew": 377.7804, "rew_std": 61.193, "Agent": "REINFORCE"}, {"env_step": 2949120, "rew": 393.2933, "rew_std": 47.9778, "Agent": "REINFORCE"}, {"env_step": 2979840, "rew": 376.6218, "rew_std": 34.4225, "Agent": "REINFORCE"}, {"env_step": 3010560, "rew": 402.7766, "rew_std": 51.016, "Agent": "REINFORCE"}, {"env_step": 3041280, "rew": 397.0908, "rew_std": 52.792, "Agent": "REINFORCE"}, {"env_step": 3072000, "rew": 367.9954, "rew_std": 50.4738, "Agent": "REINFORCE"}, {"env_step": 3102720, "rew": 393.0217, "rew_std": 40.7415, "Agent": "REINFORCE"}, {"env_step": 3133440, "rew": 382.9788, "rew_std": 73.7203, "Agent": "REINFORCE"}, {"env_step": 3164160, "rew": 392.1084, "rew_std": 80.7669, "Agent": "REINFORCE"}, {"env_step": 3194880, "rew": 368.0044, "rew_std": 109.1821, "Agent": "REINFORCE"}, {"env_step": 3225600, "rew": 365.6822, "rew_std": 102.6338, "Agent": "REINFORCE"}, {"env_step": 3256320, "rew": 382.1389, "rew_std": 90.5947, "Agent": "REINFORCE"}, {"env_step": 3287040, "rew": 369.5934, "rew_std": 119.2678, "Agent": "REINFORCE"}, {"env_step": 3317760, "rew": 393.8103, "rew_std": 75.0153, "Agent": "REINFORCE"}, {"env_step": 3348480, "rew": 395.4096, "rew_std": 55.3204, "Agent": "REINFORCE"}, {"env_step": 3379200, "rew": 397.228, "rew_std": 45.0803, "Agent": "REINFORCE"}, {"env_step": 3409920, "rew": 375.64, "rew_std": 70.8521, "Agent": "REINFORCE"}, {"env_step": 3440640, "rew": 365.7149, "rew_std": 95.5683, "Agent": "REINFORCE"}, {"env_step": 3471360, "rew": 403.0538, "rew_std": 87.3548, "Agent": "REINFORCE"}, {"env_step": 3502080, "rew": 380.4232, "rew_std": 69.2007, "Agent": "REINFORCE"}, {"env_step": 3532800, "rew": 365.7752, "rew_std": 60.142, "Agent": "REINFORCE"}, {"env_step": 3563520, "rew": 412.5866, "rew_std": 75.7224, "Agent": "REINFORCE"}, {"env_step": 3594240, "rew": 381.8078, "rew_std": 49.9293, "Agent": "REINFORCE"}, {"env_step": 3624960, "rew": 365.5222, "rew_std": 71.6263, "Agent": "REINFORCE"}, {"env_step": 3655680, "rew": 377.7163, "rew_std": 91.3498, "Agent": "REINFORCE"}, {"env_step": 3686400, "rew": 383.1321, "rew_std": 85.0593, "Agent": "REINFORCE"}, {"env_step": 3717120, "rew": 383.9464, "rew_std": 66.19, "Agent": "REINFORCE"}, {"env_step": 3747840, "rew": 385.9448, "rew_std": 67.9891, "Agent": "REINFORCE"}, {"env_step": 3778560, "rew": 400.476, "rew_std": 72.4173, "Agent": "REINFORCE"}, {"env_step": 3809280, "rew": 388.3522, "rew_std": 75.5529, "Agent": "REINFORCE"}, {"env_step": 3840000, "rew": 370.6014, "rew_std": 51.1635, "Agent": "REINFORCE"}, {"env_step": 3870720, "rew": 375.0117, "rew_std": 51.0137, "Agent": "REINFORCE"}, {"env_step": 3901440, "rew": 379.3578, "rew_std": 86.254, "Agent": "REINFORCE"}, {"env_step": 3932160, "rew": 344.8633, "rew_std": 88.2011, "Agent": "REINFORCE"}, {"env_step": 3962880, "rew": 374.3863, "rew_std": 95.5118, "Agent": "REINFORCE"}, {"env_step": 3993600, "rew": 365.4688, "rew_std": 36.3682, "Agent": "REINFORCE"}, {"env_step": 4024320, "rew": 366.4785, "rew_std": 71.8855, "Agent": "REINFORCE"}, {"env_step": 4055040, "rew": 391.0318, "rew_std": 87.9838, "Agent": "REINFORCE"}, {"env_step": 4085760, "rew": 381.0298, "rew_std": 134.4287, "Agent": "REINFORCE"}, {"env_step": 4116480, "rew": 376.8203, "rew_std": 122.3133, "Agent": "REINFORCE"}, {"env_step": 4147200, "rew": 385.0486, "rew_std": 122.6676, "Agent": "REINFORCE"}, {"env_step": 4177920, "rew": 392.2119, "rew_std": 84.2356, "Agent": "REINFORCE"}, {"env_step": 4208640, "rew": 390.4859, "rew_std": 102.2499, "Agent": "REINFORCE"}, {"env_step": 4239360, "rew": 375.6258, "rew_std": 86.9314, "Agent": "REINFORCE"}, {"env_step": 4270080, "rew": 360.4221, "rew_std": 83.6774, "Agent": "REINFORCE"}, {"env_step": 4300800, "rew": 391.1976, "rew_std": 126.0342, "Agent": "REINFORCE"}, {"env_step": 4331520, "rew": 400.9994, "rew_std": 130.515, "Agent": "REINFORCE"}, {"env_step": 4362240, "rew": 390.3708, "rew_std": 130.4103, "Agent": "REINFORCE"}, {"env_step": 4392960, "rew": 418.1074, "rew_std": 143.7854, "Agent": "REINFORCE"}, {"env_step": 4423680, "rew": 389.2711, "rew_std": 127.9304, "Agent": "REINFORCE"}, {"env_step": 4454400, "rew": 400.3435, "rew_std": 93.977, "Agent": "REINFORCE"}, {"env_step": 4485120, "rew": 392.0837, "rew_std": 70.1953, "Agent": "REINFORCE"}, {"env_step": 4515840, "rew": 384.8207, "rew_std": 52.8932, "Agent": "REINFORCE"}, {"env_step": 4546560, "rew": 388.1618, "rew_std": 70.1572, "Agent": "REINFORCE"}, {"env_step": 4577280, "rew": 404.4836, "rew_std": 32.4901, "Agent": "REINFORCE"}, {"env_step": 4608000, "rew": 369.5814, "rew_std": 48.7199, "Agent": "REINFORCE"}, {"env_step": 4638720, "rew": 389.792, "rew_std": 71.4531, "Agent": "REINFORCE"}, {"env_step": 4669440, "rew": 398.5943, "rew_std": 44.3371, "Agent": "REINFORCE"}, {"env_step": 4700160, "rew": 402.502, "rew_std": 96.5453, "Agent": "REINFORCE"}, {"env_step": 4730880, "rew": 396.9333, "rew_std": 91.251, "Agent": "REINFORCE"}, {"env_step": 4761600, "rew": 417.5429, "rew_std": 78.3296, "Agent": "REINFORCE"}, {"env_step": 4792320, "rew": 387.9348, "rew_std": 104.9392, "Agent": "REINFORCE"}, {"env_step": 4823040, "rew": 380.8827, "rew_std": 73.3557, "Agent": "REINFORCE"}, {"env_step": 4853760, "rew": 373.8808, "rew_std": 85.8424, "Agent": "REINFORCE"}, {"env_step": 4884480, "rew": 407.9777, "rew_std": 105.0307, "Agent": "REINFORCE"}, {"env_step": 4915200, "rew": 402.8455, "rew_std": 66.1868, "Agent": "REINFORCE"}, {"env_step": 4945920, "rew": 404.205, "rew_std": 58.5461, "Agent": "REINFORCE"}, {"env_step": 4976640, "rew": 386.6169, "rew_std": 125.2089, "Agent": "REINFORCE"}, {"env_step": 5007360, "rew": 408.7262, "rew_std": 123.4934, "Agent": "REINFORCE"}, {"env_step": 5038080, "rew": 408.2545, "rew_std": 86.5406, "Agent": "REINFORCE"}, {"env_step": 5068800, "rew": 388.8189, "rew_std": 52.579, "Agent": "REINFORCE"}, {"env_step": 5099520, "rew": 394.7104, "rew_std": 99.5027, "Agent": "REINFORCE"}, {"env_step": 5130240, "rew": 401.2406, "rew_std": 136.2089, "Agent": "REINFORCE"}, {"env_step": 5160960, "rew": 398.7414, "rew_std": 79.958, "Agent": "REINFORCE"}, {"env_step": 5191680, "rew": 373.4073, "rew_std": 77.1063, "Agent": "REINFORCE"}, {"env_step": 5222400, "rew": 393.8529, "rew_std": 43.3438, "Agent": "REINFORCE"}, {"env_step": 5253120, "rew": 401.865, "rew_std": 102.3126, "Agent": "REINFORCE"}, {"env_step": 5283840, "rew": 380.1387, "rew_std": 54.8124, "Agent": "REINFORCE"}, {"env_step": 5314560, "rew": 387.9333, "rew_std": 86.9538, "Agent": "REINFORCE"}, {"env_step": 5345280, "rew": 415.7576, "rew_std": 97.4606, "Agent": "REINFORCE"}, {"env_step": 5376000, "rew": 411.729, "rew_std": 39.4573, "Agent": "REINFORCE"}, {"env_step": 5406720, "rew": 385.0198, "rew_std": 43.8433, "Agent": "REINFORCE"}, {"env_step": 5437440, "rew": 407.4051, "rew_std": 40.8758, "Agent": "REINFORCE"}, {"env_step": 5468160, "rew": 386.1591, "rew_std": 76.381, "Agent": "REINFORCE"}, {"env_step": 5498880, "rew": 376.209, "rew_std": 59.3498, "Agent": "REINFORCE"}, {"env_step": 5529600, "rew": 374.4433, "rew_std": 65.4441, "Agent": "REINFORCE"}, {"env_step": 5560320, "rew": 381.2189, "rew_std": 52.3455, "Agent": "REINFORCE"}, {"env_step": 5591040, "rew": 400.3018, "rew_std": 52.0294, "Agent": "REINFORCE"}, {"env_step": 5621760, "rew": 365.557, "rew_std": 72.3095, "Agent": "REINFORCE"}, {"env_step": 5652480, "rew": 385.0313, "rew_std": 59.146, "Agent": "REINFORCE"}, {"env_step": 5683200, "rew": 397.3092, "rew_std": 82.9393, "Agent": "REINFORCE"}, {"env_step": 5713920, "rew": 379.5796, "rew_std": 92.9322, "Agent": "REINFORCE"}, {"env_step": 5744640, "rew": 369.3897, "rew_std": 82.5958, "Agent": "REINFORCE"}, {"env_step": 5775360, "rew": 375.9339, "rew_std": 65.9221, "Agent": "REINFORCE"}, {"env_step": 5806080, "rew": 385.4407, "rew_std": 49.7583, "Agent": "REINFORCE"}, {"env_step": 5836800, "rew": 358.2405, "rew_std": 76.576, "Agent": "REINFORCE"}, {"env_step": 5867520, "rew": 379.1335, "rew_std": 85.8798, "Agent": "REINFORCE"}, {"env_step": 5898240, "rew": 390.6354, "rew_std": 55.6648, "Agent": "REINFORCE"}, {"env_step": 5928960, "rew": 407.4857, "rew_std": 53.7845, "Agent": "REINFORCE"}, {"env_step": 5959680, "rew": 389.4891, "rew_std": 62.8058, "Agent": "REINFORCE"}, {"env_step": 5990400, "rew": 378.5658, "rew_std": 85.5362, "Agent": "REINFORCE"}, {"env_step": 6021120, "rew": 421.8988, "rew_std": 90.1603, "Agent": "REINFORCE"}, {"env_step": 6051840, "rew": 391.6838, "rew_std": 84.9852, "Agent": "REINFORCE"}, {"env_step": 6082560, "rew": 396.7111, "rew_std": 93.5643, "Agent": "REINFORCE"}, {"env_step": 6113280, "rew": 402.8608, "rew_std": 80.8142, "Agent": "REINFORCE"}, {"env_step": 6144000, "rew": 391.1558, "rew_std": 99.2084, "Agent": "REINFORCE"}, {"env_step": 6174720, "rew": 380.1508, "rew_std": 96.9348, "Agent": "REINFORCE"}, {"env_step": 6205440, "rew": 398.3499, "rew_std": 71.4008, "Agent": "REINFORCE"}, {"env_step": 6236160, "rew": 393.8231, "rew_std": 67.8702, "Agent": "REINFORCE"}, {"env_step": 6266880, "rew": 370.8872, "rew_std": 85.2826, "Agent": "REINFORCE"}, {"env_step": 6297600, "rew": 434.32, "rew_std": 85.2785, "Agent": "REINFORCE"}, {"env_step": 6328320, "rew": 377.7296, "rew_std": 127.8071, "Agent": "REINFORCE"}, {"env_step": 6359040, "rew": 375.258, "rew_std": 131.5744, "Agent": "REINFORCE"}, {"env_step": 6389760, "rew": 380.6274, "rew_std": 100.9078, "Agent": "REINFORCE"}, {"env_step": 6420480, "rew": 395.2915, "rew_std": 96.0091, "Agent": "REINFORCE"}, {"env_step": 6451200, "rew": 415.9735, "rew_std": 144.4578, "Agent": "REINFORCE"}, {"env_step": 6481920, "rew": 388.7657, "rew_std": 98.0722, "Agent": "REINFORCE"}, {"env_step": 6512640, "rew": 410.147, "rew_std": 121.3785, "Agent": "REINFORCE"}, {"env_step": 6543360, "rew": 420.5058, "rew_std": 143.5157, "Agent": "REINFORCE"}, {"env_step": 6574080, "rew": 381.8966, "rew_std": 117.6944, "Agent": "REINFORCE"}, {"env_step": 6604800, "rew": 380.0762, "rew_std": 107.2386, "Agent": "REINFORCE"}, {"env_step": 6635520, "rew": 389.8562, "rew_std": 143.3065, "Agent": "REINFORCE"}, {"env_step": 6666240, "rew": 390.21, "rew_std": 106.5489, "Agent": "REINFORCE"}, {"env_step": 6696960, "rew": 440.9279, "rew_std": 148.2265, "Agent": "REINFORCE"}, {"env_step": 6727680, "rew": 401.6778, "rew_std": 126.5315, "Agent": "REINFORCE"}, {"env_step": 6758400, "rew": 406.0223, "rew_std": 116.1778, "Agent": "REINFORCE"}, {"env_step": 6789120, "rew": 359.3388, "rew_std": 81.5517, "Agent": "REINFORCE"}, {"env_step": 6819840, "rew": 402.2027, "rew_std": 128.0781, "Agent": "REINFORCE"}, {"env_step": 6850560, "rew": 375.5144, "rew_std": 100.1, "Agent": "REINFORCE"}, {"env_step": 6881280, "rew": 370.0802, "rew_std": 94.1041, "Agent": "REINFORCE"}, {"env_step": 6912000, "rew": 387.4542, "rew_std": 58.9515, "Agent": "REINFORCE"}, {"env_step": 6942720, "rew": 392.3885, "rew_std": 113.2836, "Agent": "REINFORCE"}, {"env_step": 6973440, "rew": 384.8551, "rew_std": 105.257, "Agent": "REINFORCE"}, {"env_step": 7004160, "rew": 392.4603, "rew_std": 131.9155, "Agent": "REINFORCE"}, {"env_step": 7034880, "rew": 373.492, "rew_std": 90.3701, "Agent": "REINFORCE"}, {"env_step": 7065600, "rew": 369.7354, "rew_std": 118.7059, "Agent": "REINFORCE"}, {"env_step": 7096320, "rew": 398.9831, "rew_std": 57.4786, "Agent": "REINFORCE"}, {"env_step": 7127040, "rew": 366.8138, "rew_std": 101.1937, "Agent": "REINFORCE"}, {"env_step": 7157760, "rew": 385.3933, "rew_std": 115.2182, "Agent": "REINFORCE"}, {"env_step": 7188480, "rew": 363.0968, "rew_std": 106.2384, "Agent": "REINFORCE"}, {"env_step": 7219200, "rew": 364.1249, "rew_std": 132.2329, "Agent": "REINFORCE"}, {"env_step": 7249920, "rew": 386.8129, "rew_std": 106.5349, "Agent": "REINFORCE"}, {"env_step": 7280640, "rew": 361.4476, "rew_std": 92.2265, "Agent": "REINFORCE"}, {"env_step": 7311360, "rew": 367.8644, "rew_std": 76.1768, "Agent": "REINFORCE"}, {"env_step": 7342080, "rew": 372.2277, "rew_std": 61.5203, "Agent": "REINFORCE"}, {"env_step": 7372800, "rew": 383.5695, "rew_std": 65.289, "Agent": "REINFORCE"}, {"env_step": 7403520, "rew": 376.8635, "rew_std": 111.4057, "Agent": "REINFORCE"}, {"env_step": 7434240, "rew": 364.5879, "rew_std": 100.5854, "Agent": "REINFORCE"}, {"env_step": 7464960, "rew": 368.2759, "rew_std": 99.9324, "Agent": "REINFORCE"}, {"env_step": 7495680, "rew": 381.457, "rew_std": 92.2431, "Agent": "REINFORCE"}, {"env_step": 7526400, "rew": 351.8763, "rew_std": 132.2134, "Agent": "REINFORCE"}, {"env_step": 7557120, "rew": 383.2411, "rew_std": 74.2994, "Agent": "REINFORCE"}, {"env_step": 7587840, "rew": 350.1911, "rew_std": 92.5451, "Agent": "REINFORCE"}, {"env_step": 7618560, "rew": 363.1767, "rew_std": 88.2368, "Agent": "REINFORCE"}, {"env_step": 7649280, "rew": 382.3549, "rew_std": 132.6238, "Agent": "REINFORCE"}, {"env_step": 7680000, "rew": 359.9137, "rew_std": 101.0479, "Agent": "REINFORCE"}, {"env_step": 7710720, "rew": 367.2583, "rew_std": 101.4204, "Agent": "REINFORCE"}, {"env_step": 7741440, "rew": 343.4797, "rew_std": 113.3278, "Agent": "REINFORCE"}, {"env_step": 7772160, "rew": 357.4038, "rew_std": 120.2726, "Agent": "REINFORCE"}, {"env_step": 7802880, "rew": 371.3363, "rew_std": 68.6317, "Agent": "REINFORCE"}, {"env_step": 7833600, "rew": 380.6468, "rew_std": 62.7897, "Agent": "REINFORCE"}, {"env_step": 7864320, "rew": 364.0078, "rew_std": 102.1884, "Agent": "REINFORCE"}, {"env_step": 7895040, "rew": 390.4208, "rew_std": 64.8, "Agent": "REINFORCE"}, {"env_step": 7925760, "rew": 367.6176, "rew_std": 57.5819, "Agent": "REINFORCE"}, {"env_step": 7956480, "rew": 374.8038, "rew_std": 75.8725, "Agent": "REINFORCE"}, {"env_step": 7987200, "rew": 372.3415, "rew_std": 75.1058, "Agent": "REINFORCE"}, {"env_step": 8017920, "rew": 364.4274, "rew_std": 126.2467, "Agent": "REINFORCE"}, {"env_step": 8048640, "rew": 333.2996, "rew_std": 100.0399, "Agent": "REINFORCE"}, {"env_step": 8079360, "rew": 347.481, "rew_std": 100.2056, "Agent": "REINFORCE"}, {"env_step": 8110080, "rew": 370.4996, "rew_std": 95.8974, "Agent": "REINFORCE"}, {"env_step": 8140800, "rew": 374.8886, "rew_std": 118.6889, "Agent": "REINFORCE"}, {"env_step": 8171520, "rew": 379.4134, "rew_std": 118.8234, "Agent": "REINFORCE"}, {"env_step": 8202240, "rew": 383.0012, "rew_std": 64.9941, "Agent": "REINFORCE"}, {"env_step": 8232960, "rew": 375.0866, "rew_std": 89.8378, "Agent": "REINFORCE"}, {"env_step": 8263680, "rew": 375.1638, "rew_std": 101.1116, "Agent": "REINFORCE"}, {"env_step": 8294400, "rew": 385.6165, "rew_std": 69.0373, "Agent": "REINFORCE"}, {"env_step": 8325120, "rew": 375.2498, "rew_std": 104.7292, "Agent": "REINFORCE"}, {"env_step": 8355840, "rew": 402.2431, "rew_std": 79.0858, "Agent": "REINFORCE"}, {"env_step": 8386560, "rew": 382.6706, "rew_std": 91.6997, "Agent": "REINFORCE"}, {"env_step": 8417280, "rew": 372.3715, "rew_std": 82.5468, "Agent": "REINFORCE"}, {"env_step": 8448000, "rew": 401.3655, "rew_std": 98.7974, "Agent": "REINFORCE"}, {"env_step": 8478720, "rew": 385.1743, "rew_std": 58.1178, "Agent": "REINFORCE"}, {"env_step": 8509440, "rew": 375.3559, "rew_std": 104.8732, "Agent": "REINFORCE"}, {"env_step": 8540160, "rew": 349.357, "rew_std": 93.2866, "Agent": "REINFORCE"}, {"env_step": 8570880, "rew": 376.5432, "rew_std": 69.9092, "Agent": "REINFORCE"}, {"env_step": 8601600, "rew": 382.909, "rew_std": 65.2545, "Agent": "REINFORCE"}, {"env_step": 8632320, "rew": 377.8138, "rew_std": 87.565, "Agent": "REINFORCE"}, {"env_step": 8663040, "rew": 383.207, "rew_std": 80.1754, "Agent": "REINFORCE"}, {"env_step": 8693760, "rew": 371.4133, "rew_std": 89.9161, "Agent": "REINFORCE"}, {"env_step": 8724480, "rew": 376.919, "rew_std": 69.2769, "Agent": "REINFORCE"}, {"env_step": 8755200, "rew": 379.47, "rew_std": 76.5879, "Agent": "REINFORCE"}, {"env_step": 8785920, "rew": 366.7001, "rew_std": 104.0949, "Agent": "REINFORCE"}, {"env_step": 8816640, "rew": 382.4882, "rew_std": 62.5922, "Agent": "REINFORCE"}, {"env_step": 8847360, "rew": 372.0743, "rew_std": 79.0617, "Agent": "REINFORCE"}, {"env_step": 8878080, "rew": 357.0979, "rew_std": 101.0783, "Agent": "REINFORCE"}, {"env_step": 8908800, "rew": 402.782, "rew_std": 73.662, "Agent": "REINFORCE"}, {"env_step": 8939520, "rew": 383.05, "rew_std": 95.7289, "Agent": "REINFORCE"}, {"env_step": 8970240, "rew": 398.7126, "rew_std": 49.3771, "Agent": "REINFORCE"}, {"env_step": 9000960, "rew": 342.6036, "rew_std": 102.5794, "Agent": "REINFORCE"}, {"env_step": 9031680, "rew": 393.9359, "rew_std": 55.8315, "Agent": "REINFORCE"}, {"env_step": 9062400, "rew": 358.2121, "rew_std": 98.775, "Agent": "REINFORCE"}, {"env_step": 9093120, "rew": 396.7736, "rew_std": 47.5404, "Agent": "REINFORCE"}, {"env_step": 9123840, "rew": 390.0685, "rew_std": 64.218, "Agent": "REINFORCE"}, {"env_step": 9154560, "rew": 367.538, "rew_std": 85.9344, "Agent": "REINFORCE"}, {"env_step": 9185280, "rew": 367.0394, "rew_std": 96.7543, "Agent": "REINFORCE"}, {"env_step": 9216000, "rew": 365.089, "rew_std": 97.8226, "Agent": "REINFORCE"}, {"env_step": 9246720, "rew": 394.2243, "rew_std": 83.5708, "Agent": "REINFORCE"}, {"env_step": 9277440, "rew": 374.55, "rew_std": 118.3468, "Agent": "REINFORCE"}, {"env_step": 9308160, "rew": 349.5582, "rew_std": 95.0112, "Agent": "REINFORCE"}, {"env_step": 9338880, "rew": 370.6759, "rew_std": 67.2228, "Agent": "REINFORCE"}, {"env_step": 9369600, "rew": 386.7887, "rew_std": 83.2882, "Agent": "REINFORCE"}, {"env_step": 9400320, "rew": 374.7842, "rew_std": 83.7501, "Agent": "REINFORCE"}, {"env_step": 9431040, "rew": 382.3645, "rew_std": 79.8977, "Agent": "REINFORCE"}, {"env_step": 9461760, "rew": 359.2723, "rew_std": 106.1609, "Agent": "REINFORCE"}, {"env_step": 9492480, "rew": 411.7006, "rew_std": 32.7818, "Agent": "REINFORCE"}, {"env_step": 9523200, "rew": 381.7046, "rew_std": 82.238, "Agent": "REINFORCE"}, {"env_step": 9553920, "rew": 395.8383, "rew_std": 52.7248, "Agent": "REINFORCE"}, {"env_step": 9584640, "rew": 382.9098, "rew_std": 64.35, "Agent": "REINFORCE"}, {"env_step": 9615360, "rew": 374.4013, "rew_std": 104.142, "Agent": "REINFORCE"}, {"env_step": 9646080, "rew": 380.0431, "rew_std": 80.4186, "Agent": "REINFORCE"}, {"env_step": 9676800, "rew": 380.1886, "rew_std": 81.4481, "Agent": "REINFORCE"}, {"env_step": 9707520, "rew": 363.2701, "rew_std": 86.7719, "Agent": "REINFORCE"}, {"env_step": 9738240, "rew": 373.5595, "rew_std": 72.1207, "Agent": "REINFORCE"}, {"env_step": 9768960, "rew": 384.6104, "rew_std": 71.9801, "Agent": "REINFORCE"}, {"env_step": 9799680, "rew": 375.595, "rew_std": 63.4577, "Agent": "REINFORCE"}, {"env_step": 9830400, "rew": 360.9498, "rew_std": 73.2914, "Agent": "REINFORCE"}, {"env_step": 9861120, "rew": 378.3045, "rew_std": 68.1014, "Agent": "REINFORCE"}, {"env_step": 9891840, "rew": 377.6049, "rew_std": 65.1815, "Agent": "REINFORCE"}, {"env_step": 9922560, "rew": 377.4226, "rew_std": 79.2419, "Agent": "REINFORCE"}, {"env_step": 9953280, "rew": 371.0989, "rew_std": 69.7, "Agent": "REINFORCE"}, {"env_step": 9984000, "rew": 395.0888, "rew_std": 64.5988, "Agent": "REINFORCE"}, {"env_step": 10014720, "rew": 380.6971, "rew_std": 66.6752, "Agent": "REINFORCE"}, {"env_step": 10045440, "rew": 369.8089, "rew_std": 82.7607, "Agent": "REINFORCE"}, {"env_step": 10076160, "rew": 359.9013, "rew_std": 90.9538, "Agent": "REINFORCE"}, {"env_step": 10106880, "rew": 369.0725, "rew_std": 62.3956, "Agent": "REINFORCE"}, {"env_step": 10137600, "rew": 392.5896, "rew_std": 50.2816, "Agent": "REINFORCE"}, {"env_step": 10168320, "rew": 361.0641, "rew_std": 109.9217, "Agent": "REINFORCE"}, {"env_step": 10199040, "rew": 352.6764, "rew_std": 79.7465, "Agent": "REINFORCE"}, {"env_step": 10229760, "rew": 352.3748, "rew_std": 82.8586, "Agent": "REINFORCE"}, {"env_step": 0, "rew": 2.8782, "rew_std": 2.4482, "Agent": "A2C"}, {"env_step": 30000, "rew": 202.0535, "rew_std": 125.0532, "Agent": "A2C"}, {"env_step": 60000, "rew": 203.6009, "rew_std": 89.9611, "Agent": "A2C"}, {"env_step": 90000, "rew": 199.7673, "rew_std": 128.6289, "Agent": "A2C"}, {"env_step": 120000, "rew": 357.4458, "rew_std": 234.692, "Agent": "A2C"}, {"env_step": 150000, "rew": 478.5225, "rew_std": 222.9604, "Agent": "A2C"}, {"env_step": 180000, "rew": 548.6189, "rew_std": 294.5077, "Agent": "A2C"}, {"env_step": 210000, "rew": 645.4596, "rew_std": 310.3026, "Agent": "A2C"}, {"env_step": 240000, "rew": 628.3285, "rew_std": 199.1911, "Agent": "A2C"}, {"env_step": 270000, "rew": 671.2689, "rew_std": 335.6988, "Agent": "A2C"}, {"env_step": 300000, "rew": 547.9335, "rew_std": 236.2914, "Agent": "A2C"}, {"env_step": 330000, "rew": 507.6304, "rew_std": 160.0392, "Agent": "A2C"}, {"env_step": 360000, "rew": 570.0976, "rew_std": 255.8385, "Agent": "A2C"}, {"env_step": 390000, "rew": 556.3672, "rew_std": 204.2733, "Agent": "A2C"}, {"env_step": 420000, "rew": 561.311, "rew_std": 233.6207, "Agent": "A2C"}, {"env_step": 450000, "rew": 634.9585, "rew_std": 468.7015, "Agent": "A2C"}, {"env_step": 480000, "rew": 513.6519, "rew_std": 255.8916, "Agent": "A2C"}, {"env_step": 510000, "rew": 566.4153, "rew_std": 369.6174, "Agent": "A2C"}, {"env_step": 540000, "rew": 635.222, "rew_std": 356.6058, "Agent": "A2C"}, {"env_step": 570000, "rew": 664.1834, "rew_std": 377.3301, "Agent": "A2C"}, {"env_step": 600000, "rew": 782.6997, "rew_std": 539.4781, "Agent": "A2C"}, {"env_step": 630000, "rew": 719.1225, "rew_std": 355.298, "Agent": "A2C"}, {"env_step": 660000, "rew": 646.7552, "rew_std": 410.0789, "Agent": "A2C"}, {"env_step": 690000, "rew": 726.9089, "rew_std": 287.1523, "Agent": "A2C"}, {"env_step": 720000, "rew": 854.5273, "rew_std": 491.4004, "Agent": "A2C"}, {"env_step": 750000, "rew": 939.489, "rew_std": 746.9804, "Agent": "A2C"}, {"env_step": 780000, "rew": 993.2047, "rew_std": 737.1577, "Agent": "A2C"}, {"env_step": 810000, "rew": 983.7853, "rew_std": 534.7566, "Agent": "A2C"}, {"env_step": 840000, "rew": 828.3384, "rew_std": 438.1715, "Agent": "A2C"}, {"env_step": 870000, "rew": 1091.6377, "rew_std": 709.1839, "Agent": "A2C"}, {"env_step": 900000, "rew": 869.1863, "rew_std": 371.6148, "Agent": "A2C"}, {"env_step": 930000, "rew": 1028.6036, "rew_std": 618.1783, "Agent": "A2C"}, {"env_step": 960000, "rew": 1071.456, "rew_std": 608.5518, "Agent": "A2C"}, {"env_step": 990000, "rew": 1218.9367, "rew_std": 715.4378, "Agent": "A2C"}, {"env_step": 1020000, "rew": 1171.0587, "rew_std": 515.9813, "Agent": "A2C"}, {"env_step": 1050000, "rew": 1289.9904, "rew_std": 756.4703, "Agent": "A2C"}, {"env_step": 1080000, "rew": 1244.8827, "rew_std": 661.7623, "Agent": "A2C"}, {"env_step": 1110000, "rew": 1324.0092, "rew_std": 746.3179, "Agent": "A2C"}, {"env_step": 1140000, "rew": 1329.5089, "rew_std": 890.096, "Agent": "A2C"}, {"env_step": 1170000, "rew": 1440.4462, "rew_std": 868.7743, "Agent": "A2C"}, {"env_step": 1200000, "rew": 1452.9277, "rew_std": 881.4349, "Agent": "A2C"}, {"env_step": 1230000, "rew": 1641.462, "rew_std": 1019.5863, "Agent": "A2C"}, {"env_step": 1260000, "rew": 1672.4479, "rew_std": 984.9939, "Agent": "A2C"}, {"env_step": 1290000, "rew": 1608.7213, "rew_std": 864.3207, "Agent": "A2C"}, {"env_step": 1320000, "rew": 1617.7651, "rew_std": 1087.2479, "Agent": "A2C"}, {"env_step": 1350000, "rew": 1487.876, "rew_std": 988.1688, "Agent": "A2C"}, {"env_step": 1380000, "rew": 1476.9841, "rew_std": 768.3042, "Agent": "A2C"}, {"env_step": 1410000, "rew": 1805.3581, "rew_std": 1055.8783, "Agent": "A2C"}, {"env_step": 1440000, "rew": 1700.9757, "rew_std": 1165.7021, "Agent": "A2C"}, {"env_step": 1470000, "rew": 1771.6212, "rew_std": 901.2632, "Agent": "A2C"}, {"env_step": 1500000, "rew": 1766.9292, "rew_std": 967.6392, "Agent": "A2C"}, {"env_step": 1530000, "rew": 1574.3781, "rew_std": 961.4481, "Agent": "A2C"}, {"env_step": 1560000, "rew": 1788.8526, "rew_std": 792.947, "Agent": "A2C"}, {"env_step": 1590000, "rew": 1710.4779, "rew_std": 873.5761, "Agent": "A2C"}, {"env_step": 1620000, "rew": 1641.1858, "rew_std": 911.4718, "Agent": "A2C"}, {"env_step": 1650000, "rew": 1694.8277, "rew_std": 951.1382, "Agent": "A2C"}, {"env_step": 1680000, "rew": 1522.7366, "rew_std": 1068.8433, "Agent": "A2C"}, {"env_step": 1710000, "rew": 1394.1974, "rew_std": 715.0987, "Agent": "A2C"}, {"env_step": 1740000, "rew": 1350.849, "rew_std": 904.2712, "Agent": "A2C"}, {"env_step": 1770000, "rew": 1388.0303, "rew_std": 919.2345, "Agent": "A2C"}, {"env_step": 1800000, "rew": 1356.9308, "rew_std": 963.416, "Agent": "A2C"}, {"env_step": 1830000, "rew": 1316.883, "rew_std": 1070.2367, "Agent": "A2C"}, {"env_step": 1860000, "rew": 1272.1896, "rew_std": 924.1745, "Agent": "A2C"}, {"env_step": 1890000, "rew": 1386.0813, "rew_std": 1062.2458, "Agent": "A2C"}, {"env_step": 1920000, "rew": 1399.6613, "rew_std": 891.1578, "Agent": "A2C"}, {"env_step": 1950000, "rew": 1400.6987, "rew_std": 921.2195, "Agent": "A2C"}, {"env_step": 1980000, "rew": 1522.8818, "rew_std": 1060.9701, "Agent": "A2C"}, {"env_step": 2010000, "rew": 1395.3175, "rew_std": 1096.1453, "Agent": "A2C"}, {"env_step": 2040000, "rew": 1493.4073, "rew_std": 1334.7551, "Agent": "A2C"}, {"env_step": 2070000, "rew": 1475.0535, "rew_std": 1279.5097, "Agent": "A2C"}, {"env_step": 2100000, "rew": 1351.965, "rew_std": 1146.0997, "Agent": "A2C"}, {"env_step": 2130000, "rew": 1348.5182, "rew_std": 1050.743, "Agent": "A2C"}, {"env_step": 2160000, "rew": 1459.639, "rew_std": 1002.1804, "Agent": "A2C"}, {"env_step": 2190000, "rew": 1518.2845, "rew_std": 1217.019, "Agent": "A2C"}, {"env_step": 2220000, "rew": 1433.3372, "rew_std": 1185.8109, "Agent": "A2C"}, {"env_step": 2250000, "rew": 1504.5862, "rew_std": 1188.7119, "Agent": "A2C"}, {"env_step": 2280000, "rew": 1224.1892, "rew_std": 1016.6736, "Agent": "A2C"}, {"env_step": 2310000, "rew": 1372.7436, "rew_std": 1260.412, "Agent": "A2C"}, {"env_step": 2340000, "rew": 1396.0995, "rew_std": 1169.063, "Agent": "A2C"}, {"env_step": 2370000, "rew": 1319.1138, "rew_std": 1026.9624, "Agent": "A2C"}, {"env_step": 2400000, "rew": 1425.5124, "rew_std": 1166.84, "Agent": "A2C"}, {"env_step": 2430000, "rew": 1476.8554, "rew_std": 1220.7322, "Agent": "A2C"}, {"env_step": 2460000, "rew": 1365.7804, "rew_std": 1146.2047, "Agent": "A2C"}, {"env_step": 2490000, "rew": 1500.8564, "rew_std": 1190.1618, "Agent": "A2C"}, {"env_step": 2520000, "rew": 1516.4647, "rew_std": 1178.6812, "Agent": "A2C"}, {"env_step": 2550000, "rew": 1602.7566, "rew_std": 1381.7503, "Agent": "A2C"}, {"env_step": 2580000, "rew": 1364.5163, "rew_std": 1027.3753, "Agent": "A2C"}, {"env_step": 2610000, "rew": 1531.1848, "rew_std": 1143.1932, "Agent": "A2C"}, {"env_step": 2640000, "rew": 1437.8663, "rew_std": 1099.3151, "Agent": "A2C"}, {"env_step": 2670000, "rew": 1585.5412, "rew_std": 1187.9854, "Agent": "A2C"}, {"env_step": 2700000, "rew": 1463.7896, "rew_std": 939.225, "Agent": "A2C"}, {"env_step": 2730000, "rew": 1581.0281, "rew_std": 1204.4173, "Agent": "A2C"}, {"env_step": 2760000, "rew": 1582.5326, "rew_std": 1240.9568, "Agent": "A2C"}, {"env_step": 2790000, "rew": 1670.3598, "rew_std": 1185.5812, "Agent": "A2C"}, {"env_step": 2820000, "rew": 1517.0635, "rew_std": 1063.1808, "Agent": "A2C"}, {"env_step": 2850000, "rew": 1577.4279, "rew_std": 1179.4943, "Agent": "A2C"}, {"env_step": 2880000, "rew": 1549.6955, "rew_std": 1181.2401, "Agent": "A2C"}, {"env_step": 2910000, "rew": 1517.3998, "rew_std": 1060.0786, "Agent": "A2C"}, {"env_step": 2940000, "rew": 1465.4266, "rew_std": 1144.2754, "Agent": "A2C"}, {"env_step": 2970000, "rew": 1619.9052, "rew_std": 1184.9169, "Agent": "A2C"}, {"env_step": 3000000, "rew": 1658.422, "rew_std": 1153.6737, "Agent": "A2C"}] \ No newline at end of file diff --git a/examples/mujoco/gen_json.py b/examples/mujoco/gen_json.py new file mode 100755 index 0000000..0c0b113 --- /dev/null +++ b/examples/mujoco/gen_json.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 + +import os +import csv +import sys +import json + + +def merge(rootdir): + """format: $rootdir/$algo/*.csv""" + result = [] + for path, dirnames, filenames in os.walk(rootdir): + filenames = [f for f in filenames if f.endswith('.csv')] + if len(filenames) == 0: + continue + elif len(filenames) != 1: + print(f'More than 1 csv found in {path}!') + continue + algo = os.path.relpath(path, rootdir).upper() + reader = csv.DictReader(open(os.path.join(path, filenames[0]))) + for row in reader: + result.append({ + 'env_step': int(row['env_step']), + 'rew': float(row['rew']), + 'rew_std': float(row['rew:shaded']), + 'Agent': algo, + }) + open(os.path.join(rootdir, 'result.json'), 'w').write(json.dumps(result)) + + +if __name__ == "__main__": + merge(sys.argv[-1]) diff --git a/examples/mujoco/mujoco_npg.py b/examples/mujoco/mujoco_npg.py old mode 100644 new mode 100755 diff --git a/examples/mujoco/mujoco_trpo.py b/examples/mujoco/mujoco_trpo.py old mode 100644 new mode 100755 diff --git a/examples/mujoco/plotter.py b/examples/mujoco/plotter.py index 7454871..4ecd530 100755 --- a/examples/mujoco/plotter.py +++ b/examples/mujoco/plotter.py @@ -2,14 +2,12 @@ import re import os -import csv import argparse import numpy as np import matplotlib.pyplot as plt import matplotlib.ticker as mticker -from collections import defaultdict -from tools import find_all_files +from tools import find_all_files, group_files, csv2numpy def smooth(y, radius, mode='two_sided', valid_only=False): @@ -64,24 +62,6 @@ COLORS = ([ ]) -def csv2numpy(csv_file): - csv_dict = defaultdict(list) - reader = csv.DictReader(open(csv_file)) - for row in reader: - for k, v in row.items(): - csv_dict[k].append(eval(v)) - return {k: np.array(v) for k, v in csv_dict.items()} - - -def group_files(file_list, pattern): - res = defaultdict(list) - for f in file_list: - match = re.search(pattern, f) - key = match.group() if match else '' - res[key].append(f) - return res - - def plot_ax( ax, file_lists, diff --git a/examples/mujoco/tools.py b/examples/mujoco/tools.py index c4ff6c3..9e49206 100755 --- a/examples/mujoco/tools.py +++ b/examples/mujoco/tools.py @@ -6,11 +6,11 @@ import csv import tqdm import argparse import numpy as np -from typing import Dict, List, Union +from collections import defaultdict from tensorboard.backend.event_processing import event_accumulator -def find_all_files(root_dir: str, pattern: re.Pattern) -> List[str]: +def find_all_files(root_dir, pattern): """Find all files under root_dir according to relative pattern.""" file_list = [] for dirname, _, files in os.walk(root_dir): @@ -21,9 +21,25 @@ def find_all_files(root_dir: str, pattern: re.Pattern) -> List[str]: return file_list -def convert_tfevents_to_csv( - root_dir: str, refresh: bool = False -) -> Dict[str, np.ndarray]: +def group_files(file_list, pattern): + res = defaultdict(list) + for f in file_list: + match = re.search(pattern, f) + key = match.group() if match else '' + res[key].append(f) + return res + + +def csv2numpy(csv_file): + csv_dict = defaultdict(list) + reader = csv.DictReader(open(csv_file)) + for row in reader: + for k, v in row.items(): + csv_dict[k].append(eval(v)) + return {k: np.array(v) for k, v in csv_dict.items()} + + +def convert_tfevents_to_csv(root_dir, refresh=False): """Recursively convert test/rew from all tfevent file under root_dir to csv. This function assumes that there is at most one tfevents file in each directory @@ -60,11 +76,7 @@ def convert_tfevents_to_csv( return result -def merge_csv( - csv_files: List[List[Union[str, int, float]]], - root_dir: str, - remove_zero: bool = False, -) -> None: +def merge_csv(csv_files, root_dir, remove_zero=False): """Merge result in csv_files into a single csv file.""" assert len(csv_files) > 0 if remove_zero: @@ -88,13 +100,14 @@ def merge_csv( if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument('--root-dir', type=str) parser.add_argument( '--refresh', action="store_true", help="Re-generate all csv files instead of using existing one.") parser.add_argument( '--remove-zero', action="store_true", help="Remove the data point of env_step == 0.") + parser.add_argument('--root-dir', type=str) args = parser.parse_args() + csv_files = convert_tfevents_to_csv(args.root_dir, args.refresh) merge_csv(csv_files, args.root_dir, args.remove_zero)